Formatting produced by Black is commonly agreed one, and it
simplifies life a lot.
Add black to your pre-commit hook.
Signed-off-by: Matěj Cepl <mcepl@cepl.eu>
---
I was trying to be conservative (particularly changing all ' into
" makes huge diffs), but if you don't care, I can go all the way
in.
What do you think?
import_issues.py | 477 ++++++++++++++++++++++++-----------------------
1 file changed, 248 insertions(+), 229 deletions(-)
diff --git a/import_issues.py b/import_issues.py
index 666644e..d301c1d 100755
--- a/import_issues.py
+++ b/import_issues.py
@@ -136,10 +136,12 @@ from pathlib import Path
from typing import Dict, List, Optional
-ID_RE = re.compile(r'^[0-9]+$')
+ID_RE = re.compile(r"^[0-9]+$")
-logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
- level=logging.DEBUG)
+logging.basicConfig(
+ format="%(levelname)s:%(funcName)s:%(message)s",
+ level=logging.DEBUG,
+)
log = logging.getLogger()
email_count = 0
@@ -150,30 +152,32 @@ def read_id_map_file(file_path: Path) -> Dict[int, str]:
"""Reads a CSV file with ID,NAME mappings and returns the resulting dict."""
result: Dict[int, str] = {}
- with open(file_path, newline='') as fh:
+ with open(file_path, newline="") as fh:
reader = csv.reader(fh)
line_num = 0
for row in reader:
line_num += 1
- assert len(row) == 2 and ID_RE.search(row[0]) and row[1], \
- f"Row {line_num} of {file_path} is not in the form <ID>,<NAME>: {row!r}"
+ assert (
+ len(row) == 2 and ID_RE.search(row[0]) and row[1]
+ ), f"Row {line_num} of {file_path} is not in the form <ID>,<NAME>: {row!r}"
new_id = int(row[0])
- assert new_id not in result, \
- f"ID {new_id} appears multiple times in {file_path}."
+ assert (
+ new_id not in result
+ ), f"ID {new_id} appears multiple times in {file_path}."
result[new_id] = row[1]
return result
def do_mail(
- *,
- smtp,
- smtp_delay: float,
- mode: str,
- frm: str,
- to: str,
- body: str,
- subject: Optional[str] = None,
+ *,
+ smtp,
+ smtp_delay: float,
+ mode: str,
+ frm: str,
+ to: str,
+ body: str,
+ subject: Optional[str] = None,
):
global email_count
email_count += 1
@@ -217,23 +221,23 @@ def do_mail(
def open_ticket(
- *,
- smtp,
- smtp_delay: float,
- mode: str,
- srht_owner: str,
- srht_tracker: str,
- frm: str,
- title: str,
- body: str,
- created_by: Optional[str],
- created_at: str,
- closed_at: Optional[str],
- is_closed: bool,
- is_confidential: bool,
- label_names: List[str],
- milestone_name: Optional[str],
- gitlab_ticket_url: str,
+ *,
+ smtp,
+ smtp_delay: float,
+ mode: str,
+ srht_owner: str,
+ srht_tracker: str,
+ frm: str,
+ title: str,
+ body: str,
+ created_by: Optional[str],
+ created_at: str,
+ closed_at: Optional[str],
+ is_closed: bool,
+ is_confidential: bool,
+ label_names: List[str],
+ milestone_name: Optional[str],
+ gitlab_ticket_url: str,
) -> int:
global issue_count
@@ -279,14 +283,14 @@ def open_ticket(
def file_missing_ticket(
- *,
- smtp,
- smtp_delay: float,
- mode: str,
- srht_owner: str,
- srht_tracker: str,
- frm: str,
- issue_id: int,
+ *,
+ smtp,
+ smtp_delay: float,
+ mode: str,
+ srht_owner: str,
+ srht_tracker: str,
+ frm: str,
+ issue_id: int,
):
global issue_count
@@ -318,20 +322,20 @@ def file_missing_ticket(
def send_comment(
- *,
- smtp,
- smtp_delay: float,
- mode: str,
- srht_owner: str,
- srht_tracker: str,
- frm: str,
- issue_id: int,
- body: str,
- author_name: str,
- created_at: str,
- last_edited_at: str,
- is_system: bool,
- is_confidential: bool,
+ *,
+ smtp,
+ smtp_delay: float,
+ mode: str,
+ srht_owner: str,
+ srht_tracker: str,
+ frm: str,
+ issue_id: int,
+ body: str,
+ author_name: str,
+ created_at: str,
+ last_edited_at: str,
+ is_system: bool,
+ is_confidential: bool,
):
lines = []
pheaders = []
@@ -368,16 +372,16 @@ def send_comment(
def close_ticket(
- *,
- smtp,
- smtp_delay: float,
- mode: str,
- srht_owner: str,
- srht_tracker: str,
- frm: str,
- issue_id: int,
- closed_at: Optional[str],
- is_closed: bool,
+ *,
+ smtp,
+ smtp_delay: float,
+ mode: str,
+ srht_owner: str,
+ srht_tracker: str,
+ frm: str,
+ issue_id: int,
+ closed_at: Optional[str],
+ is_closed: bool,
):
lines = []
@@ -401,84 +405,81 @@ def close_ticket(
def run(
- *,
- smtp,
- smtp_delay: float,
- mode: str,
- srht_owner: str,
- srht_tracker: str,
- frm: str,
- export_dir_path: Path,
- gitlab_project_url: str,
- labels_file_path: Optional[Path],
- skip_unknown_labels: bool,
- users_file_path: Optional[Path],
- skip_unknown_users: bool,
- skip_missing_issues: bool,
- create_missing_issues: bool,
- include_confidential: bool,
- skip_confidential: bool,
+ *,
+ smtp,
+ smtp_delay: float,
+ mode: str,
+ srht_owner: str,
+ srht_tracker: str,
+ frm: str,
+ export_dir_path: Path,
+ gitlab_project_url: str,
+ labels_file_path: Optional[Path],
+ skip_unknown_labels: bool,
+ users_file_path: Optional[Path],
+ skip_unknown_users: bool,
+ skip_missing_issues: bool,
+ create_missing_issues: bool,
+ include_confidential: bool,
+ skip_confidential: bool,
):
- label_ids_to_names: Optional[Dict[int, str]] = \
+ label_ids_to_names: Optional[Dict[int, str]] = (
read_id_map_file(labels_file_path) if labels_file_path else None
- user_ids_to_names: Optional[Dict[int, str]] = \
+ )
+ user_ids_to_names: Optional[Dict[int, str]] = (
read_id_map_file(users_file_path) if users_file_path else None
+ )
# TODO Might be able to automatically map note.events.author_id to
# note.author.name for a subset of relevant users.
milestone_jsons = []
- with open(export_dir_path / 'milestones.ndjson') as milestones_file:
+ with open(export_dir_path / "milestones.ndjson") as milestones_file:
for line in milestones_file:
milestone_jsons.append(json.loads(line))
milestone_ids_to_titles = {}
for milestone_json in milestone_jsons:
- milestone_ids_to_titles[milestone_json['iid']] = milestone_json['title']
+ milestone_ids_to_titles[milestone_json["iid"]] = milestone_json["title"]
issue_jsons = []
- with open(export_dir_path / 'issues.ndjson') as issues_file:
+ with open(export_dir_path / "issues.ndjson") as issues_file:
for line in issues_file:
issue_jsons.append(json.loads(line))
if skip_confidential:
- issue_jsons = [x for x in issue_jsons if not x.get('confidential')]
+ issue_jsons = [x for x in issue_jsons if not x.get("confidential")]
for issue_json in issue_jsons:
- issue_json['notes'] = [
- n
- for n in issue_json['notes']
- if not n.get('confidential')
+ issue_json["notes"] = [
+ n for n in issue_json["notes"] if not n.get("confidential")
]
elif not include_confidential:
- have_confidential_issues = any(
- x.get('confidential')
- for x in issue_jsons
- )
+ have_confidential_issues = any(x.get("confidential") for x in issue_jsons)
have_confidential_notes = any(
- n.get('confidential')
- for x in issue_jsons
- for n in x['notes']
+ n.get("confidential") for x in issue_jsons for n in x["notes"]
)
confidential_types = []
if have_confidential_issues:
- confidential_types.append('issues')
+ confidential_types.append("issues")
if have_confidential_notes:
- confidential_types.append('notes')
- assert not (have_confidential_issues or have_confidential_notes), \
- f"Found confidential {' and '.join(confidential_types)}; please " \
- f"decide whether these should all be included, then pass either " \
- f"--include-confidential or --skip-confidential, or edit " \
+ confidential_types.append("notes")
+ assert not (have_confidential_issues or have_confidential_notes), (
+ f"Found confidential {' and '.join(confidential_types)}; please "
+ f"decide whether these should all be included, then pass either "
+ f"--include-confidential or --skip-confidential, or edit "
f"issues.ndjson for more fine-grained control."
+ )
- issue_jsons.sort(key=lambda x: x['iid'])
+ issue_jsons.sort(key=lambda x: x["iid"])
- max_issue_id = max(x['iid'] for x in issue_jsons)
- present_issue_id_set = {x['iid'] for x in issue_jsons}
+ max_issue_id = max(x["iid"] for x in issue_jsons)
+ present_issue_id_set = {x["iid"] for x in issue_jsons}
missing_issue_ids = set(range(1, max_issue_id + 1)) - present_issue_id_set
if missing_issue_ids and not (skip_missing_issues or create_missing_issues):
if skip_confidential:
- because_confidential_msg = \
+ because_confidential_msg = (
" (possibly because some confidential issues were excluded)"
+ )
else:
because_confidential_msg = ""
@@ -489,11 +490,11 @@ def run(
issues_by_id = {}
for issue_json in issue_jsons:
- issues_by_id[issue_json['iid']] = issue_json
+ issues_by_id[issue_json["iid"]] = issue_json
# Need to sort notes by date, they seem to come unsorted.
for issue_json in issue_jsons:
- issue_json['notes'].sort(key=lambda x: x['created_at'])
+ issue_json["notes"].sort(key=lambda x: x["created_at"])
log.info("-------- CREATING TICKETS")
@@ -524,16 +525,17 @@ def run(
issue_json = issues_by_id[gitlab_issue_id]
- author_id = issue_json['author_id']
+ author_id = issue_json["author_id"]
created_by: Optional[str]
if user_ids_to_names is None:
created_by = None
elif author_id in user_ids_to_names:
created_by = user_ids_to_names[author_id]
else:
- assert skip_unknown_users, \
- f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, " \
+ assert skip_unknown_users, (
+ f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, "
f"please add to the users file."
+ )
created_by = None
srht_issue_id = open_ticket(
@@ -543,57 +545,67 @@ def run(
srht_owner=srht_owner,
srht_tracker=srht_tracker,
frm=frm,
- title=issue_json['title'],
- body=issue_json['description'],
+ title=issue_json["title"],
+ body=issue_json["description"],
created_by=created_by,
- created_at=issue_json['created_at'],
- closed_at=issue_json['closed_at'],
- is_closed=(issue_json['state'] == 'closed'),
- is_confidential=(issue_json.get('confidential') is True),
- label_names=[x['label']['title'] for x in issue_json['label_links']],
- milestone_name=issue_json.get('milestone', {}).get('title') or None,
+ created_at=issue_json["created_at"],
+ closed_at=issue_json["closed_at"],
+ is_closed=(issue_json["state"] == "closed"),
+ is_confidential=(issue_json.get("confidential") is True),
+ label_names=[x["label"]["title"] for x in issue_json["label_links"]],
+ milestone_name=issue_json.get("milestone", {}).get("title") or None,
gitlab_ticket_url=f"{gitlab_project_url}/-/issues/{gitlab_issue_id}",
)
if not skip_missing_issues:
- assert srht_issue_id == gitlab_issue_id, \
- f"Internal error, srht_issue_id {srht_issue_id} != " \
- f"gitlab_issue_id {gitlab_issue_id} " \
- f"(skip_missing_issues={skip_missing_issues}, " \
+ assert srht_issue_id == gitlab_issue_id, (
+ f"Internal error, srht_issue_id {srht_issue_id} != "
+ f"gitlab_issue_id {gitlab_issue_id} "
+ f"(skip_missing_issues={skip_missing_issues}, "
f"create_missing_issues={create_missing_issues})."
+ )
issue_id_map[gitlab_issue_id] = srht_issue_id
log.info("-------- CREATING COMMENTS")
for issue_json in issue_jsons:
- for note_json in issue_json['notes']:
- system_action = note_json.get('system_note_metadata', {}).get('action', None)
+ for note_json in issue_json["notes"]:
+ system_action = note_json.get("system_note_metadata", {}).get(
+ "action", None
+ )
- body = note_json['note']
+ body = note_json["note"]
# The "Removed" part is a guess here, don't know if that actually shows up.
if label_ids_to_names is not None and (
- system_action == 'label' or re.search(r'^(Added|Removed) ~[0-9]+ label', body)
+ system_action == "label"
+ or re.search(r"^(Added|Removed) ~[0-9]+ label", body)
):
+
def expand_label(ref):
ref_num = int(ref.group(1))
if ref_num in label_ids_to_names:
return label_ids_to_names[ref_num]
- assert skip_unknown_labels, \
- f"Unknown label #{ref_num}, please add to the labels file."
+ assert (
+ skip_unknown_labels
+ ), f"Unknown label #{ref_num}, please add to the labels file."
return ref.group(0) # Return the original "~id" string.
- body = re.sub(r'~([0-9]+)', expand_label, body)
+ body = re.sub(r"~([0-9]+)", expand_label, body)
+
+ if system_action == "milestone" or re.search(
+ r"^Milestone changed to %[0-9]+$", body
+ ):
- if system_action == 'milestone' or re.search(r'^Milestone changed to %[0-9]+$', body):
def expand_milestone(ref):
ref_num = int(ref.group(1))
- assert ref_num in milestone_ids_to_titles, \
- f"Unknown milestone #{ref_num}."
+ assert (
+ ref_num in milestone_ids_to_titles
+ ), f"Unknown milestone #{ref_num}."
return milestone_ids_to_titles[ref_num]
- body = re.sub(r'%([0-9]+)', expand_milestone, body)
+ body = re.sub(r"%([0-9]+)", expand_milestone, body)
send_comment(
smtp=smtp,
@@ -602,19 +614,19 @@ def run(
srht_owner=srht_owner,
srht_tracker=srht_tracker,
frm=frm,
- issue_id=issue_id_map[issue_json['iid']],
+ issue_id=issue_id_map[issue_json["iid"]],
body=body,
- author_name=note_json['author']['name'],
- created_at=note_json['created_at'],
- last_edited_at=note_json['last_edited_at'],
- is_system=note_json['system'],
- is_confidential=(note_json['confidential'] is True),
+ author_name=note_json["author"]["name"],
+ created_at=note_json["created_at"],
+ last_edited_at=note_json["last_edited_at"],
+ is_system=note_json["system"],
+ is_confidential=(note_json["confidential"] is True),
)
log.info("-------- CLOSING CLOSED ISSUES")
for issue_json in issue_jsons:
- if issue_json['state'] == 'closed':
+ if issue_json["state"] == "closed":
close_ticket(
smtp=smtp,
smtp_delay=smtp_delay,
@@ -622,191 +634,198 @@ def run(
srht_owner=srht_owner,
srht_tracker=srht_tracker,
frm=frm,
- issue_id=issue_id_map[issue_json['iid']],
- closed_at=issue_json['closed_at'],
- is_closed=(issue_json['state'] == 'closed'),
+ issue_id=issue_id_map[issue_json["iid"]],
+ closed_at=issue_json["closed_at"],
+ is_closed=(issue_json["state"] == "closed"),
)
def main():
parser = argparse.ArgumentParser(
- prog='import_issues.py',
- description='Import Gitlab issues into Sourcehut via SMTP.',
+ prog="import_issues.py",
+ description="Import Gitlab issues into Sourcehut via SMTP.",
)
parser.add_argument(
- '--srht-owner',
+ "--srht-owner",
required=True,
- help='Owner of the Sorucehut tracker.',
+ help="Owner of the Sorucehut tracker.",
)
parser.add_argument(
- '--srht-tracker',
+ "--srht-tracker",
required=True,
- help='Name of Sourcehut tracker to submit to.',
+ help="Name of Sourcehut tracker to submit to.",
)
parser.add_argument(
- '--gitlab-project-url',
+ "--gitlab-project-url",
required=True,
help="The base URL the project on Gitlab.",
)
parser.add_argument(
- '--mode',
- default='print',
+ "--mode",
+ default="print",
help="Action to take, 'print' or 'send'.",
)
parser.add_argument(
- '--from',
+ "--from",
help="From address if mode is 'send'.",
)
parser.add_argument(
- '--smtp-host',
+ "--smtp-host",
help="SMTP host to use.",
)
parser.add_argument(
- '--smtp-port',
+ "--smtp-port",
default=None,
help="SMTP port to use.",
)
parser.add_argument(
- '--smtp-ssl',
- action='store_true',
+ "--smtp-ssl",
+ action="store_true",
help="Use SMTP over SSL.",
)
parser.add_argument(
- '--smtp-starttls',
- action='store_true',
+ "--smtp-starttls",
+ action="store_true",
help="Use STARTTLS.",
)
parser.add_argument(
- '--smtp-user',
+ "--smtp-user",
help="SMTP username.",
)
parser.add_argument(
- '--smtp-password',
+ "--smtp-password",
help="SMTP password.",
)
parser.add_argument(
- '--smtp-delay',
+ "--smtp-delay",
default=5,
help="Decimal number of seconds to wait after sending each email.",
)
parser.add_argument(
- '--labels-file',
+ "--labels-file",
help="CSV file mapping label IDs to names.",
)
parser.add_argument(
- '--skip-labels',
- action='store_true',
+ "--skip-labels",
+ action="store_true",
help="Skip mapping label IDs to names.",
)
parser.add_argument(
- '--skip-unknown-labels',
- action='store_true',
+ "--skip-unknown-labels",
+ action="store_true",
help="Skip mapping labels that aren't in the labels file.",
)
parser.add_argument(
- '--users-file',
+ "--users-file",
help="CSV file mapping user IDs to names.",
)
parser.add_argument(
- '--skip-users',
- action='store_true',
+ "--skip-users",
+ action="store_true",
help="Skip mapping user IDs to names.",
)
parser.add_argument(
- '--skip-unknown-users',
- action='store_true',
+ "--skip-unknown-users",
+ action="store_true",
help="Skip mapping users that aren't in the users file.",
)
parser.add_argument(
- '--skip-missing-issues',
- action='store_true',
+ "--skip-missing-issues",
+ action="store_true",
help="Skip missing Gitlab issue IDs; GL and sr.ht IDs will not match.",
)
parser.add_argument(
- '--create-missing-issues',
- action='store_true',
+ "--create-missing-issues",
+ action="store_true",
help="Create missing GL issues in sr.ht to make issue IDs match.",
)
parser.add_argument(
- '--include-confidential',
- action='store_true',
+ "--include-confidential",
+ action="store_true",
help="Include confidential tickets and notes.",
)
parser.add_argument(
- '--skip-confidential',
- action='store_true',
+ "--skip-confidential",
+ action="store_true",
help="Skip confidential tickets and notes.",
)
parser.add_argument(
- 'export_dir',
- help='Exported Gitlab tree/project/ directory containing ndjson files.',
+ "export_dir",
+ help="Exported Gitlab tree/project/ directory containing ndjson files.",
)
args = vars(parser.parse_args())
- export_dir = args['export_dir']
+ export_dir = args["export_dir"]
assert export_dir, f"Must have a exported project directory."
export_dir_path = Path(export_dir)
- assert export_dir_path.is_dir(), \
- f"Project directory is not a directory: {export_dir_path}"
-
- mode = args['mode']
- frm = args['from']
-
- labels_file = args['labels_file']
- skip_labels = args['skip_labels']
- skip_unknown_labels = args['skip_unknown_labels']
- assert labels_file or skip_labels, \
- f"One of --labels-file or --skip-labels must be provided."
-
- users_file = args['users_file']
- skip_users = args['skip_users']
- skip_unknown_users = args['skip_unknown_users']
- assert skip_users or users_file, \
- f"One of --users-file or --skip-users must be provided."
-
- skip_missing_issues = args['skip_missing_issues']
- create_missing_issues = args['create_missing_issues']
- assert not (skip_missing_issues and create_missing_issues), \
- f"Can accept at most one of --skip-missing-issues and --create-missing-issues."
-
- include_confidential = args['include_confidential']
- skip_confidential = args['skip_confidential']
- assert not (include_confidential and skip_confidential), \
- f"Can accept at most one of --include-confidential and --skip-confidential."
-
- if mode == 'print':
+ assert (
+ export_dir_path.is_dir()
+ ), f"Project directory is not a directory: {export_dir_path}"
+
+ mode = args["mode"]
+ frm = args["from"]
+
+ labels_file = args["labels_file"]
+ skip_labels = args["skip_labels"]
+ skip_unknown_labels = args["skip_unknown_labels"]
+ assert (
+ labels_file or skip_labels
+ ), f"One of --labels-file or --skip-labels must be provided."
+
+ users_file = args["users_file"]
+ skip_users = args["skip_users"]
+ skip_unknown_users = args["skip_unknown_users"]
+ assert (
+ skip_users or users_file
+ ), f"One of --users-file or --skip-users must be provided."
+
+ skip_missing_issues = args["skip_missing_issues"]
+ create_missing_issues = args["create_missing_issues"]
+ assert not (
+ skip_missing_issues and create_missing_issues
+ ), f"Can accept at most one of --skip-missing-issues and --create-missing-issues."
+
+ include_confidential = args["include_confidential"]
+ skip_confidential = args["skip_confidential"]
+ assert not (
+ include_confidential and skip_confidential
+ ), f"Can accept at most one of --include-confidential and --skip-confidential."
+
+ if mode == "print":
smtp = None
- elif mode == 'send':
- smtp_ssl = args['smtp_ssl']
- smtp_starttls = args['smtp_starttls']
- smtp_host = args['smtp_host'] or os.environ.get('SMTP_HOST', 'localhost')
- smtp_port = args['smtp_port'] or os.environ.get('SMTP_PORT', 465 if smtp_ssl else 25)
- smtp_user = args['smtp_user'] or os.environ.get('SMTP_USER', None)
- smtp_password = args['smtp_password'] or os.environ.get('SMTP_PASSWORD', None)
+ elif mode == "send":
+ smtp_ssl = args["smtp_ssl"]
+ smtp_starttls = args["smtp_starttls"]
+ smtp_host = args["smtp_host"] or os.environ.get("SMTP_HOST", "localhost")
+ smtp_port = args["smtp_port"] or os.environ.get(
+ "SMTP_PORT", 465 if smtp_ssl else 25
+ )
+ smtp_user = args["smtp_user"] or os.environ.get("SMTP_USER", None)
+ smtp_password = args["smtp_password"] or os.environ.get("SMTP_PASSWORD", None)
assert smtp_user, f"No SMTP user given."
assert smtp_password, f"No SMTP password given."
@@ -829,13 +848,13 @@ def main():
run(
smtp=smtp,
- smtp_delay=float(args['smtp_delay']),
+ smtp_delay=float(args["smtp_delay"]),
mode=mode,
- srht_owner=args['srht_owner'],
- srht_tracker=args['srht_tracker'],
+ srht_owner=args["srht_owner"],
+ srht_tracker=args["srht_tracker"],
frm=frm,
export_dir_path=export_dir_path,
- gitlab_project_url=args['gitlab_project_url'].rstrip('/'),
+ gitlab_project_url=args["gitlab_project_url"].rstrip("/"),
labels_file_path=None if skip_labels else Path(labels_file),
skip_unknown_labels=skip_unknown_labels,
users_file_path=None if skip_users else Path(users_file),
@@ -846,9 +865,9 @@ def main():
skip_confidential=skip_confidential,
)
- if mode == 'send':
+ if mode == "send":
smtp.quit()
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
--
2.44.0