Easier to send these two patches together. Feel free to split/rebase/... delthas (2): export: Support todo Use a common info.json metadata file doc/hut.1.scd | 2 - export.go | 6 ++- export/builds.go | 9 ++-- export/git.go | 22 +++++--- export/hg.go | 20 ++++--- export/iface.go | 7 +++ export/lists.go | 9 ++-- export/paste.go | 12 +++-- export/todo.go | 132 +++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 194 insertions(+), 25 deletions(-) create mode 100644 export/todo.go base-commit: 7c5b5e5aee0a033c4d4815449746de123caedcd8 -- 2.41.0
Adapted this patch for the latest hut changes, and pushed. Thanks!
Rebased and pushed, thanks!
Copy & paste the following snippet into your terminal to import this patchset into git:
curl -s https://lists.sr.ht/~emersion/hut-dev/patches/43454/mbox | git am -3Learn more about email & git
--- doc/hut.1.scd | 2 - export.go | 4 ++ export/todo.go | 103 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 export/todo.go diff --git a/doc/hut.1.scd b/doc/hut.1.scd index a87110d..d57f735 100644 --- a/doc/hut.1.scd +++ b/doc/hut.1.scd @@ -83,8 +83,6 @@ Additionally, mailing lists can be referred to by their email address. *export* <directory> Export account data. - Note, todo.sr.ht is not yet supported. - ## builds *artifacts* <ID> diff --git a/export.go b/export.go index 1c29b28..bbea516 100644 --- a/export.go +++ b/export.go @@ -48,6 +48,10 @@ func newExportCommand() *cobra.Command { lists := export.NewListsExporter(lc.Client, lc.BaseURL, lc.HTTP) exporters = append(exporters, lists) + tc := createClient("todo", cmd) + todo := export.NewTodoExporter(tc.Client, tc.BaseURL, tc.HTTP) + exporters = append(exporters, todo) + if _, ok := os.LookupEnv("SSH_AUTH_SOCK"); !ok { log.Println("Warning! SSH_AUTH_SOCK is not set in your environment.") log.Println("Using an SSH agent is advised to avoid unlocking your SSH keys repeatedly during the export.") diff --git a/export/todo.go b/export/todo.go new file mode 100644 index 0000000..71efbd4 --- /dev/null +++ b/export/todo.go @@ -0,0 +1,103 @@ +package export + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "path" + "time" + + "git.sr.ht/~emersion/gqlclient" + + "git.sr.ht/~emersion/hut/srht/todosrht" +) + +type TodoExporter struct { + client *gqlclient.Client + http *http.Client + baseURL string +} + +func NewTodoExporter(client *gqlclient.Client, baseURL string, + http *http.Client) *TodoExporter { + newHttp := *http + // XXX: Is this a sane default? + newHttp.Timeout = 10 * time.Minute + return &TodoExporter{ + client: client, + http: &newHttp, + baseURL: baseURL, + } +} + +func (ex *TodoExporter) Name() string { + return "todo.sr.ht" +} + +func (ex *TodoExporter) BaseURL() string { + return ex.baseURL +} + +func (ex *TodoExporter) Export(ctx context.Context, dir string) error { + log.Println("todo.sr.ht") + var cursor *todosrht.Cursor + var ret error + + for { + trackers, err := todosrht.Trackers(ex.client, ctx, cursor) + if err != nil { + return err + } + + for _, tracker := range trackers.Results { + base := path.Join(dir, tracker.Name) + + if err := ex.exportTracker(ctx, tracker, base); err != nil { + var pe partialError + if errors.As(err, &pe) { + ret = err + continue + } + return err + } + } + + cursor = trackers.Cursor + if cursor == nil { + break + } + } + + return ret +} + +func (ex *TodoExporter) exportTracker(ctx context.Context, tracker todosrht.Tracker, base string) error { + log.Printf("\t%s", tracker.Name) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, string(tracker.Export), nil) + if err != nil { + return err + } + resp, err := ex.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return partialError{fmt.Errorf("%s: server returned non-200 status %d", tracker.Name, resp.StatusCode)} + } + + f, err := os.Create(base + ".json.gz") + if err != nil { + return err + } + defer f.Close() + if _, err := io.Copy(f, resp.Body); err != nil { + return err + } + return nil +} -- 2.41.0
Adapted this patch for the latest hut changes, and pushed. Thanks!
--- export.go | 2 +- export/builds.go | 9 ++++++--- export/git.go | 22 +++++++++++++++------- export/hg.go | 20 ++++++++++++++------ export/iface.go | 7 +++++++ export/lists.go | 9 ++++++--- export/paste.go | 12 +++++++++--- export/todo.go | 31 ++++++++++++++++++++++++++++++- 8 files changed, 88 insertions(+), 24 deletions(-) diff --git a/export.go b/export.go index bbea516..c6136f9 100644 --- a/export.go +++ b/export.go @@ -66,7 +66,7 @@ func newExportCommand() *cobra.Command { log.Fatalf("Failed to create export directory: %s", err.Error()) } - stamp := path.Join(base, "export-stamp.json") + stamp := path.Join(base, "service.json") if _, err := os.Stat(stamp); err == nil { log.Printf("Skipping %s (already exported)", ex.Name()) continue diff --git a/export/builds.go b/export/builds.go index 0444752..8f347b5 100644 --- a/export/builds.go +++ b/export/builds.go @@ -43,7 +43,7 @@ func (ex *BuildsExporter) BaseURL() string { } type JobInfo struct { - Id int32 `json:"id"` + Info Status string `json:"status"` Note *string `json:"note,omitempty"` Tags []string `json:"tags"` @@ -91,7 +91,7 @@ func (ex *BuildsExporter) Export(ctx context.Context, dir string) error { } func (ex *BuildsExporter) exportJob(ctx context.Context, job *buildssrht.Job, base string) error { - infoPath := path.Join(base, "info.json") + infoPath := path.Join(base, infoFile) if _, err := os.Stat(infoPath); err == nil { log.Printf("\tSkipping #%d (already exists)", job.Id) return nil @@ -137,7 +137,10 @@ func (ex *BuildsExporter) exportJob(ctx context.Context, job *buildssrht.Job, ba defer file.Close() jobInfo := JobInfo{ - Id: job.Id, + Info: Info{ + Service: ex.Name(), + Name: strconv.Itoa(int(job.Id)), + }, Note: job.Note, Tags: job.Tags, Visibility: job.Visibility, diff --git a/export/git.go b/export/git.go index 9915f37..ae5f9b1 100644 --- a/export/git.go +++ b/export/git.go @@ -35,7 +35,7 @@ func (ex *GitExporter) BaseURL() string { // A subset of gitsrht.Repository which only contains the fields we want to // export (i.e. the ones filled in by the GraphQL query) type GitRepoInfo struct { - Name string `json:"name"` + Info Description *string `json:"description"` Visibility gitsrht.Visibility `json:"visibility"` } @@ -63,26 +63,34 @@ func (ex *GitExporter) Export(ctx context.Context, dir string) error { // TODO: Should we fetch & store ACLs? for _, repo := range repos.Results { - repoPath := path.Join(dir, "repos", repo.Name) - cloneURL := fmt.Sprintf("%s@%s:%s/%s", sshUser, baseURL.Host, repo.Owner.CanonicalName, repo.Name) - if _, err := os.Stat(repoPath); err == nil { + repoPath := path.Join(dir, repo.Name) + infoPath := path.Join(repoPath, infoFile) + clonePath := path.Join(repoPath, "repository.git") + if _, err := os.Stat(clonePath); err == nil { log.Printf("\tSkipping %s (already exists)", repo.Name) continue } + if err := os.MkdirAll(repoPath, 0o755); err != nil { + return err + } + cloneURL := fmt.Sprintf("%s@%s:%s/%s", sshUser, baseURL.Host, repo.Owner.CanonicalName, repo.Name) log.Printf("\tCloning %s", repo.Name) - cmd := exec.Command("git", "clone", "--mirror", cloneURL, repoPath) + cmd := exec.Command("git", "clone", "--mirror", cloneURL, clonePath) if err := cmd.Run(); err != nil { return err } repoInfo := GitRepoInfo{ - Name: repo.Name, + Info: Info{ + Service: ex.Name(), + Name: repo.Name, + }, Description: repo.Description, Visibility: repo.Visibility, } - file, err := os.Create(path.Join(repoPath, "srht.json")) + file, err := os.Create(infoPath) if err != nil { return err } diff --git a/export/hg.go b/export/hg.go index 6dd27b4..24e28cf 100644 --- a/export/hg.go +++ b/export/hg.go @@ -35,7 +35,7 @@ func (ex *HgExporter) BaseURL() string { // A subset of hgsrht.Repository which only contains the fields we want to // export (i.e. the ones filled in by the GraphQL query) type HgRepoInfo struct { - Name string `json:"name"` + Info Description *string `json:"description"` Visibility hgsrht.Visibility `json:"visibility"` } @@ -57,27 +57,35 @@ func (ex *HgExporter) Export(ctx context.Context, dir string) error { // TODO: Should we fetch & store ACLs? for _, repo := range repos.Results { - repoPath := path.Join(dir, "repos", repo.Name) + repoPath := path.Join(dir, repo.Name) + infoPath := path.Join(repoPath, infoFile) + clonePath := path.Join(repoPath, "repository.git") cloneURL := fmt.Sprintf("ssh://hg@%s/%s/%s", baseURL.Host, repo.Owner.CanonicalName, repo.Name) - if _, err := os.Stat(repoPath); err == nil { + if _, err := os.Stat(clonePath); err == nil { log.Printf("\tSkipping %s (already exists)", repo.Name) continue } + if err := os.MkdirAll(repoPath, 0o755); err != nil { + return err + } log.Printf("\tCloning %s", repo.Name) - cmd := exec.Command("hg", "clone", "-U", cloneURL, repoPath) + cmd := exec.Command("hg", "clone", "-U", cloneURL, clonePath) err := cmd.Run() if err != nil { return err } repoInfo := HgRepoInfo{ - Name: repo.Name, + Info: Info{ + Service: ex.Name(), + Name: repo.Name, + }, Description: repo.Description, Visibility: repo.Visibility, } - file, err := os.Create(path.Join(repoPath, ".hg", "srht.json")) + file, err := os.Create(infoPath) if err != nil { return err } diff --git a/export/iface.go b/export/iface.go index 42e9edd..c2404bd 100644 --- a/export/iface.go +++ b/export/iface.go @@ -2,6 +2,13 @@ package export import "context" +const infoFile = "info.json" + +type Info struct { + Service string `json:"service"` + Name string `json:"name"` +} + type Exporter interface { Name() string BaseURL() string diff --git a/export/lists.go b/export/lists.go index cb9f431..4299bc2 100644 --- a/export/lists.go +++ b/export/lists.go @@ -50,7 +50,7 @@ func (ex *ListsExporter) BaseURL() string { // A subset of listssrht.MailingList which only contains the fields we want to // export (i.e. the ones filled in by the GraphQL query) type MailingListInfo struct { - Name string `json:"name"` + Info Description *string `json:"description"` PermitMime []string `json:"permitMime"` RejectMime []string `json:"rejectMime"` @@ -93,7 +93,7 @@ func (ex *ListsExporter) Export(ctx context.Context, dir string) error { } func (ex *ListsExporter) exportList(ctx context.Context, list listssrht.MailingList, base string) error { - infoPath := path.Join(base, "info.json") + infoPath := path.Join(base, infoFile) if _, err := os.Stat(infoPath); err == nil { log.Printf("\tSkipping %s (already exists)", list.Name) return nil @@ -132,7 +132,10 @@ func (ex *ListsExporter) exportList(ctx context.Context, list listssrht.MailingL defer file.Close() listInfo := MailingListInfo{ - Name: list.Name, + Info: Info{ + Service: ex.Name(), + Name: list.Name, + }, Description: list.Description, PermitMime: list.PermitMime, RejectMime: list.RejectMime, diff --git a/export/paste.go b/export/paste.go index f24f67c..4427679 100644 --- a/export/paste.go +++ b/export/paste.go @@ -44,6 +44,7 @@ func (ex *PasteExporter) BaseURL() string { } type PasteInfo struct { + Info Visibility pastesrht.Visibility `json:"visibility"` } @@ -80,20 +81,21 @@ func (ex *PasteExporter) Export(ctx context.Context, dir string) error { func (ex *PasteExporter) exportPaste(ctx context.Context, paste *pastesrht.Paste, dir string) error { base := path.Join(dir, paste.Id) - infoPath := path.Join(dir, fmt.Sprintf("%s.json", paste.Id)) + infoPath := path.Join(base, infoFile) if _, err := os.Stat(infoPath); err == nil { log.Printf("\tSkipping %s (already exists)", paste.Id) return nil } log.Printf("\t%s", paste.Id) - if err := os.MkdirAll(base, 0o755); err != nil { + files := path.Join(base, "files") + if err := os.MkdirAll(files, 0o755); err != nil { return err } var ret error for _, file := range paste.Files { - if err := ex.exportFile(ctx, paste, base, &file); err != nil { + if err := ex.exportFile(ctx, paste, files, &file); err != nil { ret = err } } @@ -105,6 +107,10 @@ func (ex *PasteExporter) exportPaste(ctx context.Context, paste *pastesrht.Paste defer file.Close() pasteInfo := PasteInfo{ + Info: Info{ + Service: ex.Name(), + Name: paste.Id, + }, Visibility: paste.Visibility, } err = json.NewEncoder(file).Encode(&pasteInfo) diff --git a/export/todo.go b/export/todo.go index 71efbd4..8a45fc9 100644 --- a/export/todo.go +++ b/export/todo.go @@ -2,6 +2,7 @@ package export import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -42,6 +43,10 @@ func (ex *TodoExporter) BaseURL() string { return ex.baseURL } +type TrackerInfo struct { + Info +} + func (ex *TodoExporter) Export(ctx context.Context, dir string) error { log.Println("todo.sr.ht") var cursor *todosrht.Cursor @@ -76,7 +81,13 @@ func (ex *TodoExporter) Export(ctx context.Context, dir string) error { } func (ex *TodoExporter) exportTracker(ctx context.Context, tracker todosrht.Tracker, base string) error { + infoPath := path.Join(base, infoFile) + dataPath := path.Join(base, "tracker.json.gz") log.Printf("\t%s", tracker.Name) + if err := os.MkdirAll(base, 0o755); err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, string(tracker.Export), nil) if err != nil { return err @@ -91,7 +102,7 @@ func (ex *TodoExporter) exportTracker(ctx context.Context, tracker todosrht.Trac return partialError{fmt.Errorf("%s: server returned non-200 status %d", tracker.Name, resp.StatusCode)} } - f, err := os.Create(base + ".json.gz") + f, err := os.Create(dataPath) if err != nil { return err } @@ -99,5 +110,23 @@ func (ex *TodoExporter) exportTracker(ctx context.Context, tracker todosrht.Trac if _, err := io.Copy(f, resp.Body); err != nil { return err } + + file, err := os.Create(infoPath) + if err != nil { + return err + } + defer file.Close() + + trackerInfo := PasteInfo{ + Info: Info{ + Service: ex.Name(), + Name: tracker.Name, + }, + } + err = json.NewEncoder(file).Encode(&trackerInfo) + if err != nil { + return err + } + return nil } -- 2.41.0
Rebased and pushed, thanks!