So, close and yet so far! It almost works, https://todo.sr.ht/~mcepl/m2crypto/ is something which really resembles the useable thing, but it isn’t. Could you tell me why https://todo.sr.ht/~mcepl/m2crypto/223 is really not the equivalent of https://gitlab.com/m2crypto/m2crypto/-/issues/225, although it claims it to be so? Using this command line: ``` $ python3 ./import_issues.py \ --srht-owner=mcepl \ --srht-tracker=m2crypto \ --gitlab-project-url=https://gitlab.com/m2crypto/m2crypto/ \ --from='Matěj Cepl <mcepl@cepl.eu>' \ --mode=hut \ --create-missing-issues \ --users-file ./users.csv \ --labels-file ./labels.csv \ --include-confidential \ ../2024-05-03_14-16-911_m2crypto_m2crypto_export/tree/project/ ``` Matěj Cepl (12): Add .gitignore. Bit reformatting fix: better logging refactor: it is a general delay, not just the SMTP one. refactor: simplify API signature feat: add 'hut' option to the current 'print' and 'send' ones. refactor: all runs of the hut cmd to special function run_hut(). feat: check existence of labels feat: have smaller delay between calls to hut than SMTP ones. feat: delayed closing of tickets. refactor: split open_ticket() into two subfunctions per modes. feat: add setting labels for individual tickets .gitignore | 2 + import_issues.py | 460 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 356 insertions(+), 106 deletions(-) create mode 100644 .gitignore -- 2.43.0
Note that I have `--create-missing-issues` there, so supposedly todo.sr.ht issue ID should be the same as the GitLab ones. Best, Matěj
On Sat, 01 Jun 2024 23:44:04 +0200 Matěj Cepl <mcepl@cepl.eu> wrote:
One of the two hard problems in Computer Science...
Could we pass through hut's stderr to the console so that we can see error messages, and capture just stdout? Checking my command below, it looks like the error message is in fact written to stderr.
I'm just testing with --mode=hut now. Cheers, Bryan
Copy & paste the following snippet into your terminal to import this patchset into git:
curl -s https://lists.sr.ht/~khumba/public-inbox/patches/52892/mbox | git am -3Learn more about email & git
From: Matěj Cepl <mcepl@cepl.eu> --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff82675 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.csv +*-log*.txt -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/import_issues.py b/import_issues.py index c1dbc21..550df54 100755 --- a/import_issues.py +++ b/import_issues.py @@ -500,7 +500,7 @@ def run( raise RuntimeError( f"Don't have all issues from 1 to {max_issue_id}{because_confidential_msg}, " - f"please pass --create-missing-issues or --skip-missing-issues to proceed." + "please pass --create-missing-issues or --skip-missing-issues to proceed." ) issues_by_id = {} @@ -516,7 +516,7 @@ def run( issue_id_map: Dict[int, int] = {} # While we're creating tickets, we can't just loop over the sorted - # issue_jsons. We have to loop over potential issue IDs and handle any that + # issue_jsons. We have to loop over potential issue IDs and handle any that # are missing as well. for gitlab_issue_id in range(1, max_issue_id + 1): if gitlab_issue_id not in issues_by_id: @@ -592,7 +592,8 @@ def run( body = note_json["note"] - # The "Removed" part is a guess here, don't know if that actually shows up. + # The "Removed" part is a guess here, + # don't know if that actually shows up. if label_ids_to_names is not None and ( system_action == "label" or re.search(r"^(Added|Removed) ~[0-9]+ label", body) -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/import_issues.py b/import_issues.py index 550df54..c2ed789 100755 --- a/import_issues.py +++ b/import_issues.py @@ -191,7 +191,7 @@ def do_mail( else: raise ValueError(f"Unhandled mode {mode!r}.") - log.info(f"{verb} email #{email_count}.") + log.info("%s email #%d.", verb, email_count) date = format_datetime(datetime.now(timezone.utc)) msg_id = make_msgid() @@ -846,7 +846,7 @@ def main(): assert smtp_user, f"No SMTP user given." assert smtp_password, f"No SMTP password given." - log.info(f"Connecting to {smtp_host}:{smtp_port}, user {smtp_user!r}.") + log.info("Connecting to %s:%d, user %r.", smtp_host, smtp_port, smtp_user) if smtp_ssl: smtp = smtplib.SMTP_SSL(host=smtp_host, port=smtp_port) -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/import_issues.py b/import_issues.py index c2ed789..a3277cf 100755 --- a/import_issues.py +++ b/import_issues.py @@ -105,7 +105,7 @@ # specified either via parameters --smtp-{host,port,user,password} or the # equivalent SMTP_{HOST,PORT,USER,PASSWORD} environment variables. Pass # --smtp-ssl to enable SSL. Also by default there is a five-second delay -# between sending emails, that you may wish to change with --smtp-delay. +# between sending emails, that you may wish to change with --delay. # # ./import_issues.py \ # --srht-owner=MY_SRHT_USER \ @@ -174,7 +174,7 @@ def read_id_map_file(file_path: Path) -> Dict[int, str]: def do_mail( *, smtp, - smtp_delay: float, + delay: float, mode: str, frm: str, to: str, @@ -226,7 +226,7 @@ def do_mail( smtp.send_message(msg) - time.sleep(smtp_delay) + time.sleep(delay) else: raise RuntimeError(f"Unknown mode: {mode!r}") @@ -235,7 +235,7 @@ def do_mail( def open_ticket( *, smtp, - smtp_delay: float, + delay: float, mode: str, srht_owner: str, srht_tracker: str, @@ -282,7 +282,7 @@ def open_ticket( do_mail( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, frm=frm, to=f"~{srht_owner}/{srht_tracker}@todo.sr.ht", @@ -297,7 +297,7 @@ def open_ticket( def file_missing_ticket( *, smtp, - smtp_delay: float, + delay: float, mode: str, srht_owner: str, srht_tracker: str, @@ -308,7 +308,7 @@ def file_missing_ticket( do_mail( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, frm=frm, to=f"~{srht_owner}/{srht_tracker}@todo.sr.ht", @@ -322,7 +322,7 @@ def file_missing_ticket( # previous issue to be processed promptly. close_ticket( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, srht_owner=srht_owner, srht_tracker=srht_tracker, @@ -336,7 +336,7 @@ def file_missing_ticket( def send_comment( *, smtp, - smtp_delay: float, + delay: float, mode: str, srht_owner: str, srht_tracker: str, @@ -375,7 +375,7 @@ def send_comment( do_mail( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, frm=frm, to=f"~{srht_owner}/{srht_tracker}/{issue_id}@todo.sr.ht", @@ -386,7 +386,7 @@ def send_comment( def close_ticket( *, smtp, - smtp_delay: float, + delay: float, mode: str, srht_owner: str, srht_tracker: str, @@ -408,7 +408,7 @@ def close_ticket( do_mail( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, frm=frm, to=f"~{srht_owner}/{srht_tracker}/{issue_id}@todo.sr.ht", @@ -419,7 +419,7 @@ def close_ticket( def run( *, smtp, - smtp_delay: float, + delay: float, mode: str, srht_owner: str, srht_tracker: str, @@ -523,7 +523,7 @@ def run( if create_missing_issues: file_missing_ticket( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, srht_owner=srht_owner, srht_tracker=srht_tracker, @@ -555,7 +555,7 @@ def run( srht_issue_id = open_ticket( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, srht_owner=srht_owner, srht_tracker=srht_tracker, @@ -625,7 +625,7 @@ def run( send_comment( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, srht_owner=srht_owner, srht_tracker=srht_tracker, @@ -645,7 +645,7 @@ def run( if issue_json["state"] == "closed": close_ticket( smtp=smtp, - smtp_delay=smtp_delay, + delay=delay, mode=mode, srht_owner=srht_owner, srht_tracker=srht_tracker, @@ -725,9 +725,9 @@ def main(): ) parser.add_argument( - "--smtp-delay", + "--delay", default=5, - help="Decimal number of seconds to wait after sending each email.", + help="Decimal number of seconds to wait between accessing the server.", ) parser.add_argument( @@ -864,7 +864,7 @@ def main(): run( smtp=smtp, - smtp_delay=float(args["smtp_delay"]), + delay=float(args["delay"]), mode=mode, srht_owner=args["srht_owner"], srht_tracker=args["srht_tracker"], -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> We don't need to concatenate srht_owner and srht_tracker again and again, when we really need only complete tracker name. --- import_issues.py | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/import_issues.py b/import_issues.py index a3277cf..10ebd93 100755 --- a/import_issues.py +++ b/import_issues.py @@ -237,8 +237,7 @@ def open_ticket( smtp, delay: float, mode: str, - srht_owner: str, - srht_tracker: str, + tracker: str, frm: str, title: str, body: str, @@ -285,7 +284,7 @@ def open_ticket( delay=delay, mode=mode, frm=frm, - to=f"~{srht_owner}/{srht_tracker}@todo.sr.ht", + to=f"{tracker}@todo.sr.ht", subject=title, body="\n".join(lines), ) @@ -299,8 +298,7 @@ def file_missing_ticket( smtp, delay: float, mode: str, - srht_owner: str, - srht_tracker: str, + tracker: str, frm: str, issue_id: int, ): @@ -311,7 +309,7 @@ def file_missing_ticket( delay=delay, mode=mode, frm=frm, - to=f"~{srht_owner}/{srht_tracker}@todo.sr.ht", + to=f"{tracker}@todo.sr.ht", subject="Missing issue", body=f"Issue {issue_id} is not known.", ) @@ -324,8 +322,7 @@ def file_missing_ticket( smtp=smtp, delay=delay, mode=mode, - srht_owner=srht_owner, - srht_tracker=srht_tracker, + tracker=tracker, frm=frm, issue_id=issue_count, closed_at=None, @@ -338,8 +335,7 @@ def send_comment( smtp, delay: float, mode: str, - srht_owner: str, - srht_tracker: str, + tracker: str, frm: str, issue_id: int, body: str, @@ -378,7 +374,7 @@ def send_comment( delay=delay, mode=mode, frm=frm, - to=f"~{srht_owner}/{srht_tracker}/{issue_id}@todo.sr.ht", + to=f"{tracker}/{issue_id}@todo.sr.ht", body="\n".join(lines), ) @@ -388,8 +384,7 @@ def close_ticket( smtp, delay: float, mode: str, - srht_owner: str, - srht_tracker: str, + tracker: str, frm: str, issue_id: int, closed_at: Optional[str], @@ -411,7 +406,7 @@ def close_ticket( delay=delay, mode=mode, frm=frm, - to=f"~{srht_owner}/{srht_tracker}/{issue_id}@todo.sr.ht", + to=f"{tracker}/{issue_id}@todo.sr.ht", body="\n".join(lines), ) @@ -421,8 +416,7 @@ def run( smtp, delay: float, mode: str, - srht_owner: str, - srht_tracker: str, + tracker: str, frm: str, export_dir_path: Path, gitlab_project_url: str, @@ -525,8 +519,7 @@ def run( smtp=smtp, delay=delay, mode=mode, - srht_owner=srht_owner, - srht_tracker=srht_tracker, + tracker=tracker, frm=frm, issue_id=gitlab_issue_id, ) @@ -557,8 +550,7 @@ def run( smtp=smtp, delay=delay, mode=mode, - srht_owner=srht_owner, - srht_tracker=srht_tracker, + tracker=tracker, frm=frm, title=issue_json["title"], body=issue_json["description"], @@ -627,8 +619,7 @@ def run( smtp=smtp, delay=delay, mode=mode, - srht_owner=srht_owner, - srht_tracker=srht_tracker, + tracker=tracker, frm=frm, issue_id=issue_id_map[issue_json["iid"]], body=body, @@ -647,8 +638,7 @@ def run( smtp=smtp, delay=delay, mode=mode, - srht_owner=srht_owner, - srht_tracker=srht_tracker, + tracker=tracker, frm=frm, issue_id=issue_id_map[issue_json["iid"]], closed_at=issue_json["closed_at"], @@ -831,6 +821,10 @@ def main(): include_confidential and skip_confidential ), f"Can accept at most one of --include-confidential and --skip-confidential." + srht_owner = args["srht_owner"] + srht_tracker = args["srht_tracker"] + tracker = f"~{srht_owner}/{srht_tracker}" + if mode == "print": smtp = None elif mode == "send": @@ -866,8 +860,7 @@ def main(): smtp=smtp, delay=float(args["delay"]), mode=mode, - srht_owner=args["srht_owner"], - srht_tracker=args["srht_tracker"], + tracker=tracker, frm=frm, export_dir_path=export_dir_path, gitlab_project_url=args["gitlab_project_url"].rstrip("/"), -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 152 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 116 insertions(+), 36 deletions(-) diff --git a/import_issues.py b/import_issues.py index 10ebd93..1b9b14e 100755 --- a/import_issues.py +++ b/import_issues.py @@ -127,6 +127,7 @@ import logging import json import os import re +import subprocess import smtplib import sys import time @@ -279,15 +280,35 @@ def open_ticket( lines.append("") lines.append(body) - do_mail( - smtp=smtp, - delay=delay, - mode=mode, - frm=frm, - to=f"{tracker}@todo.sr.ht", - subject=title, - body="\n".join(lines), - ) + if mode in ["send", "print"]: + do_mail( + smtp=smtp, + delay=delay, + mode=mode, + frm=frm, + to=f"{tracker}@todo.sr.ht", + subject=title, + body="\n".join(lines), + ) + elif mode == "hut": + msg = title + if len(lines): + msg += "\n" + "\n".join(lines) + res = subprocess.run( + [ + "hut", + "todo", + "ticket", + "create", + "-t", + tracker, + "--stdin", + ], + encoding="utf-8", + input=msg, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) issue_count += 1 return issue_count @@ -304,15 +325,33 @@ def file_missing_ticket( ): global issue_count - do_mail( - smtp=smtp, - delay=delay, - mode=mode, - frm=frm, - to=f"{tracker}@todo.sr.ht", - subject="Missing issue", - body=f"Issue {issue_id} is not known.", - ) + if mode in ["send", "print"]: + do_mail( + smtp=smtp, + delay=delay, + mode=mode, + frm=frm, + to=f"{tracker}@todo.sr.ht", + subject="Missing issue", + body=f"Issue {issue_id} is not known.", + ) + elif mode == "hut": + msg = f"Missing issue\n\nIssue {issue_id} is not known." + res = subprocess.run( + [ + "hut", + "todo", + "ticket", + "create", + "-t", + tracker, + "--stdin", + ], + encoding="utf-8", + input=msg, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) issue_count += 1 @@ -369,14 +408,32 @@ def send_comment( lines.append("") lines.append(f"(Last edited at {last_edited_at}.)") - do_mail( - smtp=smtp, - delay=delay, - mode=mode, - frm=frm, - to=f"{tracker}/{issue_id}@todo.sr.ht", - body="\n".join(lines), - ) + if mode in ["send", "print"]: + do_mail( + smtp=smtp, + delay=delay, + mode=mode, + frm=frm, + to=f"{tracker}/{issue_id}@todo.sr.ht", + body="\n".join(lines), + ) + elif mode == "hut": + res = subprocess.run( + [ + "hut", + "todo", + "ticket", + "comment", + "-t", + tracker, + "--stdin", + issue_id, + ], + encoding="utf-8", + input="\n".join(lines), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) def close_ticket( @@ -398,17 +455,39 @@ def close_ticket( elif is_closed: lines.append("Ticket closed.") - lines.append("") - lines.append("!resolve fixed") + if mode in ["send", "print"]: + lines.append("") + lines.append("!resolve fixed") - do_mail( - smtp=smtp, - delay=delay, - mode=mode, - frm=frm, - to=f"{tracker}/{issue_id}@todo.sr.ht", - body="\n".join(lines), - ) + do_mail( + smtp=smtp, + delay=delay, + mode=mode, + frm=frm, + to=f"{tracker}/{issue_id}@todo.sr.ht", + body="\n".join(lines), + ) + + elif mode == "hut": + res = subprocess.run( + [ + "hut", + "todo", + "ticket", + "update-status", + "--resolution", + "fixed", + "--status", + "closed", + "-t", + tracker, + issue_id, + ], + encoding="utf-8", + input="\n".join(lines), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) def run( @@ -672,6 +751,7 @@ def main(): parser.add_argument( "--mode", + choices=["print", "send", "hut"], default="print", help="Action to take, 'print' or 'send'.", ) -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 90 +++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 66 deletions(-) diff --git a/import_issues.py b/import_issues.py index 1b9b14e..556e861 100755 --- a/import_issues.py +++ b/import_issues.py @@ -233,6 +233,19 @@ def do_mail( raise RuntimeError(f"Unknown mode: {mode!r}") +def run_hut(cmds, tracker, msg, args=None): + if args is None: + args = [] + res = subprocess.run( + ["hut", "todo"] + cmds + ["-t", tracker] + args, + encoding="utf-8", + input=msg, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return res + + def open_ticket( *, smtp, @@ -294,21 +307,7 @@ def open_ticket( msg = title if len(lines): msg += "\n" + "\n".join(lines) - res = subprocess.run( - [ - "hut", - "todo", - "ticket", - "create", - "-t", - tracker, - "--stdin", - ], - encoding="utf-8", - input=msg, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + run_hut(["ticket", "create", "--stdin"], tracker, msg) issue_count += 1 return issue_count @@ -337,21 +336,7 @@ def file_missing_ticket( ) elif mode == "hut": msg = f"Missing issue\n\nIssue {issue_id} is not known." - res = subprocess.run( - [ - "hut", - "todo", - "ticket", - "create", - "-t", - tracker, - "--stdin", - ], - encoding="utf-8", - input=msg, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + run_hut(["ticket", "create"], tracker, msg, ["--stdin"]) issue_count += 1 @@ -408,6 +393,8 @@ def send_comment( lines.append("") lines.append(f"(Last edited at {last_edited_at}.)") + body = "\n".join(lines) + if mode in ["send", "print"]: do_mail( smtp=smtp, @@ -415,25 +402,9 @@ def send_comment( mode=mode, frm=frm, to=f"{tracker}/{issue_id}@todo.sr.ht", - body="\n".join(lines), ) elif mode == "hut": - res = subprocess.run( - [ - "hut", - "todo", - "ticket", - "comment", - "-t", - tracker, - "--stdin", - issue_id, - ], - encoding="utf-8", - input="\n".join(lines), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + run_hut(["ticket", "comment"], tracker, None, ["--stdin", issue_id]) def close_ticket( @@ -469,24 +440,11 @@ def close_ticket( ) elif mode == "hut": - res = subprocess.run( - [ - "hut", - "todo", - "ticket", - "update-status", - "--resolution", - "fixed", - "--status", - "closed", - "-t", - tracker, - issue_id, - ], - encoding="utf-8", - input="\n".join(lines), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + run_hut( + ["ticket", "update-status"], + tracker, + None, + ["--resolution", "fixed", "--status", "closed", issue_id], ) @@ -753,7 +711,7 @@ def main(): "--mode", choices=["print", "send", "hut"], default="print", - help="Action to take, 'print' or 'send'.", + help="Action to take.", ) parser.add_argument( -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> use `hut graphql` instead --- import_issues.py | 63 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/import_issues.py b/import_issues.py index 556e861..c900ead 100755 --- a/import_issues.py +++ b/import_issues.py @@ -123,8 +123,8 @@ import argparse import csv -import logging import json +import logging import os import re import subprocess @@ -137,14 +137,44 @@ from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional - -ID_RE = re.compile(r"^[0-9]+$") - logging.basicConfig( format="%(levelname)s:%(funcName)s:%(message)s", level=logging.DEBUG, stream=sys.stdout, ) + +ID_RE = re.compile(r"^[0-9]+$") + + +def get_labels(tracker: str) -> list[dict[str, str]]: + """ + collects labels for your named tracker + + param: tracker: name of the tracker + return: list of all labels in the tracker + """ + query = ( + 'query { me { tracker(name: "' + + tracker + + '") { labels { results { id, name, foregroundColor, backgroundColor, created } } } }}' + ) + + try: + ret = subprocess.run( + ["hut", "graphql", "todo", "--stdin"], + input=query, + text=True, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as ex: + raise RuntimeError( + f"hut failed with excitcode {ex.returncode} and stderr:\n{ex.stderr}" + ) from ex + data = json.loads(ret.stdout) + return data["me"]["tracker"]["labels"]["results"] + + log = logging.getLogger() email_count = 0 @@ -448,6 +478,17 @@ def close_ticket( ) +def ensure_label(tracker: str, name: str, bg_color: str, fg_color: str = "#FFFFFF"): + labels = get_labels(tracker.split("/", 1)[1]) + if not ([x for x in labels if x["name"] == name]): + run_hut( + ["label", "create"], + tracker, + None, + ["--background", bg_color, "--foreground", fg_color, name], + ) + + def run( *, smtp, @@ -824,7 +865,7 @@ def main(): args = vars(parser.parse_args()) export_dir = args["export_dir"] - assert export_dir, f"Must have a exported project directory." + assert export_dir, "Must have a exported project directory." export_dir_path = Path(export_dir) assert ( export_dir_path.is_dir() @@ -838,26 +879,26 @@ def main(): skip_unknown_labels = args["skip_unknown_labels"] assert ( labels_file or skip_labels - ), f"One of --labels-file or --skip-labels must be provided." + ), "One of --labels-file or --skip-labels must be provided." users_file = args["users_file"] skip_users = args["skip_users"] skip_unknown_users = args["skip_unknown_users"] assert ( skip_users or users_file - ), f"One of --users-file or --skip-users must be provided." + ), "One of --users-file or --skip-users must be provided." skip_missing_issues = args["skip_missing_issues"] create_missing_issues = args["create_missing_issues"] assert not ( skip_missing_issues and create_missing_issues - ), f"Can accept at most one of --skip-missing-issues and --create-missing-issues." + ), "Can accept at most one of --skip-missing-issues and --create-missing-issues." include_confidential = args["include_confidential"] skip_confidential = args["skip_confidential"] assert not ( include_confidential and skip_confidential - ), f"Can accept at most one of --include-confidential and --skip-confidential." + ), "Can accept at most one of --include-confidential and --skip-confidential." srht_owner = args["srht_owner"] srht_tracker = args["srht_tracker"] @@ -875,8 +916,8 @@ def main(): smtp_user = args["smtp_user"] or os.environ.get("SMTP_USER", None) smtp_password = args["smtp_password"] or os.environ.get("SMTP_PASSWORD", None) - assert smtp_user, f"No SMTP user given." - assert smtp_password, f"No SMTP password given." + assert smtp_user, "No SMTP user given." + assert smtp_password, "No SMTP password given." log.info("Connecting to %s:%d, user %r.", smtp_host, smtp_port, smtp_user) -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 63 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/import_issues.py b/import_issues.py index c900ead..b4b4232 100755 --- a/import_issues.py +++ b/import_issues.py @@ -263,16 +263,32 @@ def do_mail( raise RuntimeError(f"Unknown mode: {mode!r}") -def run_hut(cmds, tracker, msg, args=None): +def run_hut(cmds, tracker, msg, args=None, delay=None): + log.debug( + f"run_hut: cmds = {cmds}, tracker = {tracker}, args = {args}\n\nmsg:\n{msg}" + ) if args is None: args = [] - res = subprocess.run( - ["hut", "todo"] + cmds + ["-t", tracker] + args, - encoding="utf-8", - input=msg, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + if msg is None: + res = subprocess.run( + ["hut", "todo"] + cmds + ["-t", tracker] + args, + check=True, + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + else: + res = subprocess.run( + ["hut", "todo"] + cmds + ["-t", tracker, "--stdin"] + args, + check=True, + encoding="utf-8", + input=msg, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + time.sleep(delay) + return res @@ -337,7 +353,7 @@ def open_ticket( msg = title if len(lines): msg += "\n" + "\n".join(lines) - run_hut(["ticket", "create", "--stdin"], tracker, msg) + run_hut(["ticket", "create"], tracker, msg, delay=delay) issue_count += 1 return issue_count @@ -366,7 +382,7 @@ def file_missing_ticket( ) elif mode == "hut": msg = f"Missing issue\n\nIssue {issue_id} is not known." - run_hut(["ticket", "create"], tracker, msg, ["--stdin"]) + run_hut(["ticket", "create"], tracker, msg, delay=delay) issue_count += 1 @@ -434,7 +450,7 @@ def send_comment( to=f"{tracker}/{issue_id}@todo.sr.ht", ) elif mode == "hut": - run_hut(["ticket", "comment"], tracker, None, ["--stdin", issue_id]) + run_hut(["ticket", "comment"], tracker, body, [str(issue_id)], delay=delay) def close_ticket( @@ -474,11 +490,14 @@ def close_ticket( ["ticket", "update-status"], tracker, None, - ["--resolution", "fixed", "--status", "closed", issue_id], + [str(issue_id), "--resolution", "fixed", "--status", "resolved"], + delay=delay, ) -def ensure_label(tracker: str, name: str, bg_color: str, fg_color: str = "#FFFFFF"): +def ensure_label( + tracker: str, name: str, bg_color: str, fg_color: str = "#FFFFFF", delay=None +): labels = get_labels(tracker.split("/", 1)[1]) if not ([x for x in labels if x["name"] == name]): run_hut( @@ -486,6 +505,7 @@ def ensure_label(tracker: str, name: str, bg_color: str, fg_color: str = "#FFFFF tracker, None, ["--background", bg_color, "--foreground", fg_color, name], + delay=delay, ) @@ -795,7 +815,7 @@ def main(): parser.add_argument( "--delay", - default=5, + default=None, help="Decimal number of seconds to wait between accessing the server.", ) @@ -904,9 +924,18 @@ def main(): srht_tracker = args["srht_tracker"] tracker = f"~{srht_owner}/{srht_tracker}" - if mode == "print": + delay = args["delay"] + if delay is None: + if mode == "hut": + delay = 0.5 + else: + delay = 5 + else: + delay = float(delay) + + if mode != "send": smtp = None - elif mode == "send": + else: smtp_ssl = args["smtp_ssl"] smtp_starttls = args["smtp_starttls"] smtp_host = args["smtp_host"] or os.environ.get("SMTP_HOST", "localhost") @@ -937,7 +966,7 @@ def main(): run( smtp=smtp, - delay=float(args["delay"]), + delay=delay, mode=mode, tracker=tracker, frm=frm, -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> Delaying should hopefully help todo.sr.ht to process all previous changes before the tickets are closed. --- import_issues.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/import_issues.py b/import_issues.py index b4b4232..b919112 100755 --- a/import_issues.py +++ b/import_issues.py @@ -145,6 +145,8 @@ logging.basicConfig( ID_RE = re.compile(r"^[0-9]+$") +tickets_to_be_closed = [] + def get_labels(tracker: str) -> list[dict[str, str]]: """ @@ -386,17 +388,15 @@ def file_missing_ticket( issue_count += 1 - # TODO Send these emails at the end, so that there isn't such a need for the - # previous issue to be processed promptly. - close_ticket( - smtp=smtp, - delay=delay, - mode=mode, - tracker=tracker, - frm=frm, - issue_id=issue_count, - closed_at=None, - is_closed=False, # Save one line of text. + tickets_to_be_closed.append( + ( + smtp, + delay, + mode, + tracker, + frm, + issue_count, + ) ) @@ -743,6 +743,19 @@ def run( is_closed=(issue_json["state"] == "closed"), ) + log.info("Delayed closing issues.") + for ticket in tickets_to_be_closed: + close_ticket( + smtp=ticket[0], + delay=ticket[1], + mode=ticket[2], + tracker=ticket[3], + frm=ticket[4], + issue_id=ticket[5], + closed_at=None, + is_closed=False, + ) + def main(): parser = argparse.ArgumentParser( -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 136 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 117 insertions(+), 19 deletions(-) diff --git a/import_issues.py b/import_issues.py index b919112..64e1a91 100755 --- a/import_issues.py +++ b/import_issues.py @@ -133,6 +133,7 @@ import sys import time from email.message import EmailMessage from email.utils import format_datetime, make_msgid +from functools import cache from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional @@ -148,6 +149,7 @@ ID_RE = re.compile(r"^[0-9]+$") tickets_to_be_closed = [] +@cache def get_labels(tracker: str) -> list[dict[str, str]]: """ collects labels for your named tracker @@ -267,7 +269,7 @@ def do_mail( def run_hut(cmds, tracker, msg, args=None, delay=None): log.debug( - f"run_hut: cmds = {cmds}, tracker = {tracker}, args = {args}\n\nmsg:\n{msg}" + "cmds = %s, tracker = %s, args = %s\n\nmsg:\n%s", cmds, tracker, args, msg ) if args is None: args = [] @@ -294,8 +296,55 @@ def run_hut(cmds, tracker, msg, args=None, delay=None): return res -def open_ticket( - *, +def open_ticket_by_hut( + delay: float, + tracker: str, + frm: str, + title: str, + body: str, + created_by: Optional[str], + created_at: str, + closed_at: Optional[str], + is_closed: bool, + is_confidential: bool, + labels: List[Dict[str, Any]], + milestone_name: Optional[str], + gitlab_ticket_url: str, +) -> str: + + lines = [] + pheaders = [] + + pheaders.append(f"Migrated from: {gitlab_ticket_url}") + + if created_by: + pheaders.append(f"Created by: {created_by}") + pheaders.append(f"Created at: {created_at}") + + if closed_at is not None: + pheaders.append(f"Closed at: {closed_at}") + elif is_closed: + pheaders.append("State: closed") + + if milestone_name: + pheaders.append(f"Milestone: {milestone_name}") + + if is_confidential: + pheaders.append("Confidential: true") + + lines.append(" \\\n".join(pheaders)) + lines.append("") + lines.append(body) + + msg = title + if len(lines): + msg += "\n" + "\n".join(lines) + out = run_hut(["ticket", "create"], tracker, msg, delay=delay) + + return out + + +def open_ticket_by_email( smtp, delay: float, mode: str, @@ -311,9 +360,7 @@ def open_ticket( label_names: List[str], milestone_name: Optional[str], gitlab_ticket_url: str, -) -> int: - global issue_count - +): lines = [] pheaders = [] @@ -341,21 +388,71 @@ def open_ticket( lines.append("") lines.append(body) - if mode in ["send", "print"]: - do_mail( - smtp=smtp, - delay=delay, - mode=mode, - frm=frm, - to=f"{tracker}@todo.sr.ht", - subject=title, - body="\n".join(lines), + do_mail( + smtp=smtp, + delay=delay, + mode=mode, + frm=frm, + to=f"{tracker}@todo.sr.ht", + subject=title, + body="\n".join(lines), + ) + + +def open_ticket( + *, + smtp, + delay: float, + mode: str, + tracker: str, + frm: str, + title: str, + body: str, + created_by: Optional[str], + created_at: str, + closed_at: Optional[str], + is_closed: bool, + is_confidential: bool, + labels: List[Dict[str, Any]], + milestone_name: Optional[str], + gitlab_ticket_url: str, +) -> int: + global issue_count + + if mode in ("send", "print"): + open_ticket_by_email( + smtp, + delay, + mode, + tracker, + frm, + title, + body, + created_by, + created_at, + closed_at, + is_closed, + is_confidential, + labels, + milestone_name, + gitlab_ticket_url, ) elif mode == "hut": - msg = title - if len(lines): - msg += "\n" + "\n".join(lines) - run_hut(["ticket", "create"], tracker, msg, delay=delay) + open_ticket_by_hut( + delay, + tracker, + frm, + title, + body, + created_by, + created_at, + closed_at, + is_closed, + is_confidential, + labels, + milestone_name, + gitlab_ticket_url, + ) issue_count += 1 return issue_count @@ -448,6 +545,7 @@ def send_comment( mode=mode, frm=frm, to=f"{tracker}/{issue_id}@todo.sr.ht", + body=body, ) elif mode == "hut": run_hut(["ticket", "comment"], tracker, body, [str(issue_id)], delay=delay) -- 2.43.0
From: Matěj Cepl <mcepl@cepl.eu> --- import_issues.py | 97 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/import_issues.py b/import_issues.py index 64e1a91..bb183e7 100755 --- a/import_issues.py +++ b/import_issues.py @@ -136,7 +136,7 @@ from email.utils import format_datetime, make_msgid from functools import cache from datetime import datetime, timezone from pathlib import Path -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional logging.basicConfig( format="%(levelname)s:%(funcName)s:%(message)s", @@ -279,7 +279,7 @@ def run_hut(cmds, tracker, msg, args=None, delay=None): check=True, encoding="utf-8", stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT, ) else: res = subprocess.run( @@ -288,7 +288,7 @@ def run_hut(cmds, tracker, msg, args=None, delay=None): encoding="utf-8", input=msg, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT, ) time.sleep(delay) @@ -296,8 +296,10 @@ def run_hut(cmds, tracker, msg, args=None, delay=None): return res -def open_ticket_by_hut( +def open_ticket_by_email( + smtp, delay: float, + mode: str, tracker: str, frm: str, title: str, @@ -310,8 +312,7 @@ def open_ticket_by_hut( labels: List[Dict[str, Any]], milestone_name: Optional[str], gitlab_ticket_url: str, -) -> str: - +): lines = [] pheaders = [] @@ -329,6 +330,17 @@ def open_ticket_by_hut( if milestone_name: pheaders.append(f"Milestone: {milestone_name}") + if labels: + pheaders.append( + "Labels: " + + ", ".join( + [ + x["label"]["title"] + for x in sorted(labels, key=lambda x: x["label"]["title"]) + ] + ) + ) + if is_confidential: pheaders.append("Confidential: true") @@ -336,18 +348,19 @@ def open_ticket_by_hut( lines.append("") lines.append(body) - msg = title - if len(lines): - msg += "\n" + "\n".join(lines) - out = run_hut(["ticket", "create"], tracker, msg, delay=delay) - - return out + do_mail( + smtp=smtp, + delay=delay, + mode=mode, + frm=frm, + to=f"{tracker}@todo.sr.ht", + subject=title, + body="\n".join(lines), + ) -def open_ticket_by_email( - smtp, +def open_ticket_by_hut( delay: float, - mode: str, tracker: str, frm: str, title: str, @@ -357,10 +370,11 @@ def open_ticket_by_email( closed_at: Optional[str], is_closed: bool, is_confidential: bool, - label_names: List[str], + labels: List[Dict[str, Any]], milestone_name: Optional[str], gitlab_ticket_url: str, -): +) -> str: + lines = [] pheaders = [] @@ -378,9 +392,6 @@ def open_ticket_by_email( if milestone_name: pheaders.append(f"Milestone: {milestone_name}") - if label_names: - pheaders.append("Labels: " + ", ".join(sorted(label_names))) - if is_confidential: pheaders.append("Confidential: true") @@ -388,19 +399,43 @@ def open_ticket_by_email( lines.append("") lines.append(body) - do_mail( - smtp=smtp, - delay=delay, - mode=mode, - frm=frm, - to=f"{tracker}@todo.sr.ht", - subject=title, - body="\n".join(lines), - ) + msg = title + if len(lines): + msg += "\n" + "\n".join(lines) + out = run_hut(["ticket", "create"], tracker, msg, delay=delay) + + for label in sorted(labels, key=lambda x: x["label"]["title"]): + # {"target_type":"Issue", + # "created_at":"2023-11-03T22:10:29.419Z", + # "updated_at":"2023-11-03T22:10:29.419Z", + # "label":{"title":"smime", + # "color":"#00b140", + # "project_id":346279, + # "created_at":"2023-02-02T10:49:17.779Z", + # "updated_at":"2023-02-02T10:49:17.779Z", + # "template":false, + # "description":null, + # "group_id":null, + # "type":"ProjectLabel", + # "priorities":[]}} + label_name = label["label"]["title"] + label_color = label["label"]["color"] + ensure_label(tracker, label_name, label_color, delay=delay) + for line in out.stdout.splitlines(): + if line.startswith("Created new ticket"): + bug_id = int(line.strip().split("#", 1)[1]) + run_hut( + ["ticket", "label"], + tracker, + None, + args=[str(bug_id), "-l", label_name], + delay=delay, + ) + + return out def open_ticket( - *, smtp, delay: float, mode: str, @@ -755,7 +790,7 @@ def run( closed_at=issue_json["closed_at"], is_closed=(issue_json["state"] == "closed"), is_confidential=(issue_json.get("confidential") is True), - label_names=[x["label"]["title"] for x in issue_json["label_links"]], + labels=list(issue_json["label_links"]), milestone_name=issue_json.get("milestone", {}).get("title") or None, gitlab_ticket_url=f"{gitlab_project_url}/-/issues/{gitlab_issue_id}", ) -- 2.43.0
Matěj Cepl <mcepl@suse.cz>Note that I have `--create-missing-issues` there, so supposedly todo.sr.ht issue ID should be the same as the GitLab ones. Best, Matěj