delthas: 1 Add the import command 19 files changed, 542 insertions(+), 32 deletions(-)
Copy & paste the following snippet into your terminal to import this patchset into git:
curl -s https://lists.sr.ht/~emersion/hut-dev/patches/43842/mbox | git am -3Learn more about email & git
Now that export is supported, we are adding a command to import that data back to another sourcehut instance or account. The design is to have export lay out a tree of directories by service and resource, with each resource containing a info.json file with common fields (service and resource name) and possibly additional fields depending on the service. Import builds on top of that existing design by walking the file tree from the given folder, finding any info.json file, and passing the folder containing that file to the relevant importer. For example, export could write: /out/git.sr.ht/kuroneko/info.json /out/git.sr.ht/kuroneko/repository.git/... /out/meta.sr.ht/info.json /out/meta.sr.ht/ssh.keys ... Then, import started on /out would walk the tree, finding: /out/git.sr.ht/kuroneko/info.json /out/meta.sr.ht/info.json It passes "/out/git.sr.ht/kuroneko" to the git.sr.ht importer and "/out/meta.sr.ht" to the meta.sr.ht importer. import could also be started on /out/git.sr.ht to import only resources of that service, or even /out/git.sr.ht/kuroneko to import that project only. Specific care was given not to abort import in case of an error while importing a single resource, because this is typically due to a resource already existing, so we just log and skip to the next resource. This has the added benefit of being somewhat idempotent, as running import twice would skip over existing resources created over the first run. A particular exception to this are pastes, which are created with a new ID every time, and therefore are created again on a second run. --- Of note is that the resulting Export and ImportResource are not symetrical: Export export all resources of the service, while ImportResource imports a single resource. This is intended for now, as a intermediate step before enriching the export command with a way to select individual resources to export (therefore creating something similar to an ExportResource). --- This patch also modifies two exporters slightly: - meta.sr.ht now outputs an info.json file (just to let the import code automatically discover it and import SSH and PGP keys) - hg.sr.ht now outputs the repository to repo/repository(/.hg) instead of repo/repository.git(/.hg), which did not feel right ;) --- doc/hut.1.scd | 3 + export/builds.go | 6 +- export/git.go | 63 ++++++++++++++++++- export/hg.go | 64 +++++++++++++++++-- export/iface.go | 3 +- export/lists.go | 60 ++++++++++++++++-- export/meta.go | 57 ++++++++++++++++- export/paste.go | 63 ++++++++++++++++++- export/todo.go | 46 +++++++++++++- import.go | 100 ++++++++++++++++++++++++++++++ main.go | 1 + srht/gitsrht/gql.go | 6 +- srht/gitsrht/operations.graphql | 5 ++ srht/hgsrht/gql.go | 17 ++++- srht/hgsrht/operations.graphql | 18 ++++++ srht/listssrht/gql.go | 26 +++++++- srht/listssrht/operations.graphql | 18 ++++++ srht/todosrht/gql.go | 13 +++- srht/todosrht/operations.graphql | 5 ++ 19 files changed, 542 insertions(+), 32 deletions(-) create mode 100644 import.go diff --git a/doc/hut.1.scd b/doc/hut.1.scd index 753fabd..b3680ff 100644 --- a/doc/hut.1.scd +++ b/doc/hut.1.scd @@ -86,6 +86,9 @@ Additionally, mailing lists can be referred to by their email address. *export* <directory> Export account data. +*import* <directory> + Import account data. + ## builds *artifacts* <ID> diff --git a/export/builds.go b/export/builds.go index fdefcc1..3fc02cd 100644 --- a/export/builds.go +++ b/export/builds.go @@ -79,8 +79,12 @@ func (ex *BuildsExporter) Export(ctx context.Context, dir string) error { return ret } +func (ex *BuildsExporter) ImportResource(ctx context.Context, dir string) error { + panic("not implemented") +} + func (ex *BuildsExporter) exportJob(ctx context.Context, job *buildssrht.Job, base string) error { - infoPath := path.Join(base, infoFilename) + infoPath := path.Join(base, InfoFilename) if _, err := os.Stat(infoPath); err == nil { log.Printf("\tSkipping #%d (already exists)", job.Id) return nil diff --git a/export/git.go b/export/git.go index 7a79271..8ee8dd0 100644 --- a/export/git.go +++ b/export/git.go @@ -2,18 +2,22 @@ package export import ( "context" + "encoding/json" "fmt" "log" "net/url" "os" "os/exec" "path" + "strings" "git.sr.ht/~emersion/gqlclient" "git.sr.ht/~emersion/hut/srht/gitsrht" ) +const gitRepositoryDir = "repository.git" + type GitExporter struct { client *gqlclient.Client baseURL string @@ -29,6 +33,8 @@ type GitRepoInfo struct { Info Description *string `json:"description"` Visibility gitsrht.Visibility `json:"visibility"` + Readme *string `json:"readme,omitempty"` + HEAD *string `json:"HEAD,omitempty"` } func (ex *GitExporter) Export(ctx context.Context, dir string) error { @@ -53,8 +59,8 @@ func (ex *GitExporter) Export(ctx context.Context, dir string) error { // TODO: Should we fetch & store ACLs? for _, repo := range repos.Results { repoPath := path.Join(dir, repo.Name) - infoPath := path.Join(repoPath, infoFilename) - clonePath := path.Join(repoPath, "repository.git") + infoPath := path.Join(repoPath, InfoFilename) + clonePath := path.Join(repoPath, gitRepositoryDir) cloneURL := fmt.Sprintf("%s@%s:%s/%s", sshUser, baseURL.Host, repo.Owner.CanonicalName, repo.Name) if _, err := os.Stat(clonePath); err == nil { @@ -71,6 +77,11 @@ func (ex *GitExporter) Export(ctx context.Context, dir string) error { return err } + var head *string + if repo.HEAD != nil { + h := strings.TrimPrefix(repo.HEAD.Name, "refs/heads/") + head = &h + } repoInfo := GitRepoInfo{ Info: Info{ Service: "git.sr.ht", @@ -78,6 +89,8 @@ func (ex *GitExporter) Export(ctx context.Context, dir string) error { }, Description: repo.Description, Visibility: repo.Visibility, + Readme: repo.Readme, + HEAD: head, } if err := writeJSON(infoPath, &repoInfo); err != nil { return err @@ -92,3 +105,49 @@ func (ex *GitExporter) Export(ctx context.Context, dir string) error { return nil } + +func (ex *GitExporter) ImportResource(ctx context.Context, dir string) error { + settings, err := gitsrht.SshSettings(ex.client, ctx) + if err != nil { + return fmt.Errorf("failed to get Git SSH settings: %v", err) + } + sshUser := settings.Settings.SshUser + + baseURL, err := url.Parse(ex.baseURL) + if err != nil { + panic(err) + } + + infoPath := path.Join(dir, InfoFilename) + + f, err := os.Open(infoPath) + if err != nil { + return err + } + defer f.Close() + var info GitRepoInfo + if err := json.NewDecoder(f).Decode(&info); err != nil { + return err + } + + g, err := gitsrht.CreateRepository(ex.client, ctx, info.Name, info.Visibility, info.Description, nil) + if err != nil { + return fmt.Errorf("failed to create Git repository: %v", err) + } + + clonePath := path.Join(dir, gitRepositoryDir) + cloneURL := fmt.Sprintf("%s@%s:%s/%s", sshUser, baseURL.Host, g.Owner.CanonicalName, info.Name) + + cmd := exec.Command("git", "-C", clonePath, "push", "--mirror", cloneURL) + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to push Git repository: %v", err) + } + + if _, err := gitsrht.UpdateRepository(ex.client, ctx, g.Id, gitsrht.RepoInput{ + Readme: info.Readme, + HEAD: info.HEAD, + }); err != nil { + return fmt.Errorf("failed to update Git repository: %v", err) + } + return nil +} diff --git a/export/hg.go b/export/hg.go index 5e86961..3be7ddf 100644 --- a/export/hg.go +++ b/export/hg.go @@ -2,6 +2,7 @@ package export import ( "context" + "encoding/json" "fmt" "log" "net/url" @@ -14,6 +15,8 @@ import ( "git.sr.ht/~emersion/hut/srht/hgsrht" ) +const hgRepositoryDir = "repository" + type HgExporter struct { client *gqlclient.Client baseURL string @@ -27,8 +30,10 @@ func NewHgExporter(client *gqlclient.Client, baseURL string) *HgExporter { // export (i.e. the ones filled in by the GraphQL query) type HgRepoInfo struct { Info - Description *string `json:"description"` - Visibility hgsrht.Visibility `json:"visibility"` + Description *string `json:"description"` + Visibility hgsrht.Visibility `json:"visibility"` + Readme *string `json:"readme,omitempty"` + NonPublishing *bool `json:"nonPublishing,omitempty"` } func (ex *HgExporter) Export(ctx context.Context, dir string) error { @@ -47,8 +52,8 @@ func (ex *HgExporter) Export(ctx context.Context, dir string) error { // TODO: Should we fetch & store ACLs? for _, repo := range repos.Results { repoPath := path.Join(dir, repo.Name) - infoPath := path.Join(repoPath, infoFilename) - clonePath := path.Join(repoPath, "repository.git") + infoPath := path.Join(repoPath, InfoFilename) + clonePath := path.Join(repoPath, hgRepositoryDir) cloneURL := fmt.Sprintf("ssh://hg@%s/%s/%s", baseURL.Host, repo.Owner.CanonicalName, repo.Name) if _, err := os.Stat(clonePath); err == nil { @@ -66,13 +71,16 @@ func (ex *HgExporter) Export(ctx context.Context, dir string) error { return err } + nonPublishing := repo.NonPublishing repoInfo := HgRepoInfo{ Info: Info{ Service: "hg.sr.ht", Name: repo.Name, }, - Description: repo.Description, - Visibility: repo.Visibility, + Description: repo.Description, + Visibility: repo.Visibility, + Readme: repo.Readme, + NonPublishing: &nonPublishing, } if err := writeJSON(infoPath, &repoInfo); err != nil { return err @@ -87,3 +95,47 @@ func (ex *HgExporter) Export(ctx context.Context, dir string) error { return nil } + +func (ex *HgExporter) ImportResource(ctx context.Context, dir string) error { + baseURL, err := url.Parse(ex.baseURL) + if err != nil { + panic(err) + } + + infoPath := path.Join(dir, InfoFilename) + + f, err := os.Open(infoPath) + if err != nil { + return err + } + defer f.Close() + var info HgRepoInfo + if err := json.NewDecoder(f).Decode(&info); err != nil { + return err + } + + description := "" + if info.Description != nil { + description = *info.Description + } + + h, err := hgsrht.CreateRepository(ex.client, ctx, info.Name, info.Visibility, description) + if err != nil { + return fmt.Errorf("failed to create Mercurial repository: %v", err) + } + + clonePath := path.Join(dir, hgRepositoryDir) + cloneURL := fmt.Sprintf("ssh://hg@%s/%s/%s", baseURL.Host, h.Owner.CanonicalName, info.Name) + + if err := exec.Command("hg", "push", "--cwd", clonePath, cloneURL).Run(); err != nil { + return fmt.Errorf("failed to push Mercurial repository: %v", err) + } + + if _, err := hgsrht.UpdateRepository(ex.client, ctx, h.Id, hgsrht.RepoInput{ + Readme: info.Readme, + NonPublishing: info.NonPublishing, + }); err != nil { + return fmt.Errorf("failed to update Mercurial repository: %v", err) + } + return nil +} diff --git a/export/iface.go b/export/iface.go index 4089dd7..6f5aa41 100644 --- a/export/iface.go +++ b/export/iface.go @@ -6,7 +6,7 @@ import ( "os" ) -const infoFilename = "info.json" +const InfoFilename = "info.json" type Info struct { Service string `json:"service"` @@ -15,6 +15,7 @@ type Info struct { type Exporter interface { Export(ctx context.Context, dir string) error + ImportResource(ctx context.Context, dir string) error } type partialError struct { diff --git a/export/lists.go b/export/lists.go index ac15936..7173aec 100644 --- a/export/lists.go +++ b/export/lists.go @@ -2,6 +2,7 @@ package export import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -16,6 +17,8 @@ import ( "git.sr.ht/~emersion/hut/srht/listssrht" ) +const archiveFile = "archive.mbox" + type ListsExporter struct { client *gqlclient.Client http *http.Client @@ -39,9 +42,10 @@ func NewListsExporter(client *gqlclient.Client, http *http.Client) *ListsExporte // export (i.e. the ones filled in by the GraphQL query) type MailingListInfo struct { Info - Description *string `json:"description"` - PermitMime []string `json:"permitMime"` - RejectMime []string `json:"rejectMime"` + Description *string `json:"description"` + Visibility listssrht.Visibility `json:"visibility"` + PermitMime []string `json:"permitMime"` + RejectMime []string `json:"rejectMime"` } func (ex *ListsExporter) Export(ctx context.Context, dir string) error { @@ -79,8 +83,24 @@ func (ex *ListsExporter) Export(ctx context.Context, dir string) error { return ret } +func (ex *ListsExporter) ImportResource(ctx context.Context, dir string) error { + infoPath := path.Join(dir, InfoFilename) + + f, err := os.Open(infoPath) + if err != nil { + return err + } + defer f.Close() + var info MailingListInfo + if err := json.NewDecoder(f).Decode(&info); err != nil { + return err + } + + return ex.importList(ctx, info, dir) +} + func (ex *ListsExporter) exportList(ctx context.Context, list listssrht.MailingList, base string) error { - infoPath := path.Join(base, infoFilename) + infoPath := path.Join(base, InfoFilename) if _, err := os.Stat(infoPath); err == nil { log.Printf("\tSkipping %s (already exists)", list.Name) return nil @@ -103,7 +123,7 @@ func (ex *ListsExporter) exportList(ctx context.Context, list listssrht.MailingL list.Name, resp.StatusCode)} } - archive, err := os.Create(path.Join(base, "archive.mbox")) + archive, err := os.Create(path.Join(base, archiveFile)) if err != nil { return err } @@ -118,6 +138,7 @@ func (ex *ListsExporter) exportList(ctx context.Context, list listssrht.MailingL Name: list.Name, }, Description: list.Description, + Visibility: list.Visibility, PermitMime: list.PermitMime, RejectMime: list.RejectMime, } @@ -127,3 +148,32 @@ func (ex *ListsExporter) exportList(ctx context.Context, list listssrht.MailingL return nil } + +func (ex *ListsExporter) importList(ctx context.Context, list MailingListInfo, base string) error { + archive, err := os.Open(path.Join(base, archiveFile)) + if err != nil { + return err + } + defer archive.Close() + + l, err := listssrht.CreateMailingList(ex.client, ctx, list.Name, list.Description, list.Visibility) + if err != nil { + return fmt.Errorf("failed to create mailing list: %v", err) + } + + if _, err := listssrht.UpdateMailingList(ex.client, ctx, l.Id, listssrht.MailingListInput{ + PermitMime: list.PermitMime, + RejectMime: list.RejectMime, + }); err != nil { + return fmt.Errorf("failed to update mailing list: %v", err) + } + + if _, err := listssrht.ImportMailingListSpool(ex.client, ctx, l.Id, gqlclient.Upload{ + Filename: archiveFile, + MIMEType: "application/mbox", + Body: archive, + }); err != nil { + return fmt.Errorf("failed to import mailing list emails: %v", err) + } + return nil +} diff --git a/export/meta.go b/export/meta.go index c4975bd..029afa9 100644 --- a/export/meta.go +++ b/export/meta.go @@ -1,8 +1,10 @@ package export import ( + "bufio" "context" "fmt" + "log" "os" "path" @@ -11,6 +13,9 @@ import ( "git.sr.ht/~emersion/hut/srht/metasrht" ) +const sshKeysFile = "ssh.keys" +const pgpKeysFile = "keys.pgp" + type MetaExporter struct { client *gqlclient.Client } @@ -30,7 +35,7 @@ func (ex *MetaExporter) Export(ctx context.Context, dir string) error { var cursor *metasrht.Cursor - sshFile, err := os.Create(path.Join(dir, "ssh.keys")) + sshFile, err := os.Create(path.Join(dir, sshKeysFile)) if err != nil { return err } @@ -54,7 +59,7 @@ func (ex *MetaExporter) Export(ctx context.Context, dir string) error { } } - pgpFile, err := os.Create(path.Join(dir, "keys.pgp")) + pgpFile, err := os.Create(path.Join(dir, pgpKeysFile)) if err != nil { return err } @@ -78,5 +83,53 @@ func (ex *MetaExporter) Export(ctx context.Context, dir string) error { } } + if err := writeJSON(path.Join(dir, InfoFilename), &Info{ + Service: "meta.sr.ht", + Name: me.CanonicalName, + }); err != nil { + return err + } + + return nil +} + +func (ex *MetaExporter) ImportResource(ctx context.Context, dir string) error { + sshFile, err := os.Open(path.Join(dir, sshKeysFile)) + if err != nil { + return err + } + defer sshFile.Close() + sshScanner := bufio.NewScanner(sshFile) + for sshScanner.Scan() { + if sshScanner.Text() == "" { + continue + } + if _, err := metasrht.CreateSSHKey(ex.client, ctx, sshScanner.Text()); err != nil { + log.Printf("Error importing SSH key: %v", err) + continue + } + } + if sshScanner.Err() != nil { + return err + } + + pgpFile, err := os.Open(path.Join(dir, pgpKeysFile)) + if err != nil { + return err + } + defer pgpFile.Close() + pgpScanner := bufio.NewScanner(pgpFile) + for pgpScanner.Scan() { + if pgpScanner.Text() == "" { + continue
This won't work for PGP keys since they can contain newlines - unlike SSH keys. As a naive solution maybe we can split on something like "-----BEGIN PGP PUBLIC KEY BLOCK-----"? If we cannot find a nice way to parse those from a file, we could just save each key in a dedicated file as well. Besides this the patch looks good in a first review.
+ } + if _, err := metasrht.CreatePGPKey(ex.client, ctx, pgpScanner.Text()); err != nil { + log.Printf("Error importing PGP key: %v", err) + continue + } + } + if pgpScanner.Err() != nil { + return err + } return nil } diff --git a/export/paste.go b/export/paste.go index 867a494..51d70e7 100644 --- a/export/paste.go +++ b/export/paste.go @@ -2,6 +2,7 @@ package export import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -16,6 +17,8 @@ import ( "git.sr.ht/~emersion/hut/srht/pastesrht" ) +const filesDir = "files" + type PasteExporter struct { client *gqlclient.Client http *http.Client @@ -66,16 +69,32 @@ func (ex *PasteExporter) Export(ctx context.Context, dir string) error { return ret } +func (ex *PasteExporter) ImportResource(ctx context.Context, dir string) error { + infoPath := path.Join(dir, InfoFilename) + + f, err := os.Open(infoPath) + if err != nil { + return err + } + defer f.Close() + var info PasteInfo + if err := json.NewDecoder(f).Decode(&info); err != nil { + return err + } + + return ex.importPaste(ctx, info, dir) +} + func (ex *PasteExporter) exportPaste(ctx context.Context, paste *pastesrht.Paste, dir string) error { base := path.Join(dir, paste.Id) - infoPath := path.Join(base, infoFilename) + infoPath := path.Join(base, InfoFilename) if _, err := os.Stat(infoPath); err == nil { log.Printf("\tSkipping %s (already exists)", paste.Id) return nil } log.Printf("\t%s", paste.Id) - files := path.Join(base, "files") + files := path.Join(base, filesDir) if err := os.MkdirAll(files, 0o755); err != nil { return err } @@ -101,6 +120,44 @@ func (ex *PasteExporter) exportPaste(ctx context.Context, paste *pastesrht.Paste return ret } +func (ex *PasteExporter) importPaste(ctx context.Context, paste PasteInfo, base string) error { + filesPath := path.Join(base, filesDir) + items, err := os.ReadDir(filesPath) + if err != nil { + return err + } + + var files []gqlclient.Upload + for _, item := range items { + if item.IsDir() { + continue + } + f, err := os.Open(path.Join(filesPath, item.Name())) + if err != nil { + return err + } + defer f.Close() + var name string + if item.Name() != paste.Name { + name = item.Name() + } + files = append(files, gqlclient.Upload{ + Filename: name, + // MIMEType is not used by the API, except for checking that it is a "text". + // Parsing the MIME type from the extension would cause issues: ".json" is parsed as "application/json", + // which gets rejected because it is not a "text/". + // Since the API does not use the type besides that, always send a dummy text value. + MIMEType: "text/plain", + Body: f, + }) + } + + if _, err := pastesrht.CreatePaste(ex.client, ctx, files, paste.Visibility); err != nil { + return fmt.Errorf("failed to create paste: %v", err) + } + return nil +} + func (ex *PasteExporter) exportFile(ctx context.Context, paste *pastesrht.Paste, base string, file *pastesrht.File) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, string(file.Contents), nil) if err != nil { @@ -113,7 +170,7 @@ func (ex *PasteExporter) exportFile(ctx context.Context, paste *pastesrht.Paste, defer resp.Body.Close() name := paste.Id - if file.Filename != nil { + if file.Filename != nil && *file.Filename != "" { name = *file.Filename } diff --git a/export/todo.go b/export/todo.go index 4512faf..cc13902 100644 --- a/export/todo.go +++ b/export/todo.go @@ -2,6 +2,7 @@ package export import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -12,10 +13,11 @@ import ( "time" "git.sr.ht/~emersion/gqlclient" - "git.sr.ht/~emersion/hut/srht/todosrht" ) +const trackerFile = "tracker.json.gz" + type TodoExporter struct { client *gqlclient.Client http *http.Client @@ -69,14 +71,30 @@ func (ex *TodoExporter) Export(ctx context.Context, dir string) error { return ret } +func (ex *TodoExporter) ImportResource(ctx context.Context, dir string) error { + infoPath := path.Join(dir, InfoFilename) + + f, err := os.Open(infoPath) + if err != nil { + return err + } + defer f.Close() + var info TrackerInfo + if err := json.NewDecoder(f).Decode(&info); err != nil { + return err + } + + return ex.importTracker(ctx, info, dir) +} + func (ex *TodoExporter) exportTracker(ctx context.Context, tracker todosrht.Tracker, base string) error { - infoPath := path.Join(base, infoFilename) + infoPath := path.Join(base, InfoFilename) if _, err := os.Stat(infoPath); err == nil { log.Printf("\tSkipping %s (already exists)", tracker.Name) return nil } - dataPath := path.Join(base, "tracker.json.gz") + dataPath := path.Join(base, trackerFile) log.Printf("\t%s", tracker.Name) if err := os.MkdirAll(base, 0o755); err != nil { return err @@ -119,3 +137,25 @@ func (ex *TodoExporter) exportTracker(ctx context.Context, tracker todosrht.Trac return nil } + +func (ex *TodoExporter) importTracker(ctx context.Context, tracker TrackerInfo, base string) error { + dataPath := path.Join(base, trackerFile) + f, err := os.Open(dataPath) + if err != nil { + return err + } + defer f.Close() + + t, err := todosrht.CreateTracker(ex.client, ctx, tracker.Name, tracker.Description, tracker.Visibility) + if err != nil { + return fmt.Errorf("failed to create issue tracker: %v", err) + } + if _, err := todosrht.ImportTrackerDump(ex.client, ctx, t.Id, gqlclient.Upload{ + Filename: trackerFile, + MIMEType: "application/gzip", + Body: f, + }); err != nil { + return fmt.Errorf("failed to import issue tracker dump: %v", err) + } + return nil +} diff --git a/import.go b/import.go new file mode 100644 index 0000000..57ff9f2 --- /dev/null +++ b/import.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/json" + "github.com/spf13/cobra" + "io/fs" + "log" + "os" + "path/filepath" + + "git.sr.ht/~emersion/hut/export" +) + +func newImportCommand() *cobra.Command { + run := func(cmd *cobra.Command, args []string) { + importers := make(map[string]export.Exporter) + + mc := createClient("meta", cmd) + meta := export.NewMetaExporter(mc.Client) + importers["meta.sr.ht"] = meta + + gc := createClient("git", cmd) + git := export.NewGitExporter(gc.Client, gc.BaseURL) + importers["git.sr.ht"] = git + + hc := createClient("hg", cmd) + hg := export.NewHgExporter(hc.Client, hc.BaseURL) + importers["hg.sr.ht"] = hg + + pc := createClient("paste", cmd) + paste := export.NewPasteExporter(pc.Client, pc.HTTP) + importers["paste.sr.ht"] = paste + + lc := createClient("lists", cmd) + lists := export.NewListsExporter(lc.Client, lc.HTTP) + importers["lists.sr.ht"] = lists + + tc := createClient("todo", cmd) + todo := export.NewTodoExporter(tc.Client, tc.HTTP) + importers["todo.sr.ht"] = todo + + if _, ok := os.LookupEnv("SSH_AUTH_SOCK"); !ok { + log.Println("Warning! SSH_AUTH_SOCK is not set in your environment.") + log.Println("Using an SSH agent is advised to avoid unlocking your SSH keys repeatedly during the import.") + } + + ctx := cmd.Context() + log.Println("Importing account data...") + + var lastService string + filepath.WalkDir(args[0], func(path string, d fs.DirEntry, err error) error { + if err != nil { + log.Printf("Error importing %q: %v", path, err) + return nil + } + if d.IsDir() { + return nil + } + if filepath.Base(path) != export.InfoFilename { + return nil + } + f, err := os.Open(path) + if err != nil { + log.Printf("Error importing %q: %v", path, err) + return filepath.SkipDir + } + defer f.Close() + var info export.Info + if err := json.NewDecoder(f).Decode(&info); err != nil { + log.Printf("Error importing %q: %v", path, err) + return filepath.SkipDir + } + importer, ok := importers[info.Service] + if !ok { + // Some services are exported but never imported. + return filepath.SkipDir + } + if lastService != info.Service { + log.Println(info.Service) + lastService = info.Service + } + log.Printf("\t%s", info.Name) + if err := importer.ImportResource(ctx, filepath.Dir(path)); err != nil { + log.Printf("Error importing %q: %v", path, err) + } + return filepath.SkipDir + }) + + log.Println("Import complete.") + } + return &cobra.Command{ + Use: "import <directory>", + Short: "Imports your account data", + Args: cobra.ExactArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return nil, cobra.ShellCompDirectiveFilterDirs + }, + Run: run, + } +} diff --git a/main.go b/main.go index 33d9422..d72b0f2 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,7 @@ func main() { cmd.AddCommand(newBuildsCommand()) cmd.AddCommand(newExportCommand()) + cmd.AddCommand(newImportCommand()) cmd.AddCommand(newGitCommand()) cmd.AddCommand(newGraphqlCommand()) cmd.AddCommand(newHgCommand()) diff --git a/srht/gitsrht/gql.go b/srht/gitsrht/gql.go index 1d918f6..3e916c8 100644 --- a/srht/gitsrht/gql.go +++ b/srht/gitsrht/gql.go @@ -655,7 +655,7 @@ func RepositoryByUser(client *gqlclient.Client, ctx context.Context, username st } func Repositories(client *gqlclient.Client, ctx context.Context, cursor *Cursor) (repositories *RepositoryCursor, err error) { - op := gqlclient.NewOperation("query repositories ($cursor: Cursor) {\n\trepositories(cursor: $cursor) {\n\t\t... repos\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t}\n\tcursor\n}\n") + op := gqlclient.NewOperation("query repositories ($cursor: Cursor) {\n\trepositories(cursor: $cursor) {\n\t\t... repos\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\treadme\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t\tHEAD {\n\t\t\tname\n\t\t}\n\t}\n\tcursor\n}\n") op.Var("cursor", cursor) var respData struct { Repositories *RepositoryCursor @@ -665,7 +665,7 @@ func Repositories(client *gqlclient.Client, ctx context.Context, cursor *Cursor) } func RepositoriesByUser(client *gqlclient.Client, ctx context.Context, username string, cursor *Cursor) (user *User, err error) { - op := gqlclient.NewOperation("query repositoriesByUser ($username: String!, $cursor: Cursor) {\n\tuser(username: $username) {\n\t\trepositories(cursor: $cursor) {\n\t\t\t... repos\n\t\t}\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t}\n\tcursor\n}\n") + op := gqlclient.NewOperation("query repositoriesByUser ($username: String!, $cursor: Cursor) {\n\tuser(username: $username) {\n\t\trepositories(cursor: $cursor) {\n\t\t\t... repos\n\t\t}\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\treadme\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t\tHEAD {\n\t\t\tname\n\t\t}\n\t}\n\tcursor\n}\n") op.Var("username", username) op.Var("cursor", cursor) var respData struct { @@ -770,7 +770,7 @@ func DeleteArtifact(client *gqlclient.Client, ctx context.Context, id int32) (de } func CreateRepository(client *gqlclient.Client, ctx context.Context, name string, visibility Visibility, description *string, cloneUrl *string) (createRepository *Repository, err error) { - op := gqlclient.NewOperation("mutation createRepository ($name: String!, $visibility: Visibility!, $description: String, $cloneUrl: String) {\n\tcreateRepository(name: $name, visibility: $visibility, description: $description, cloneUrl: $cloneUrl) {\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t\tname\n\t}\n}\n") + op := gqlclient.NewOperation("mutation createRepository ($name: String!, $visibility: Visibility!, $description: String, $cloneUrl: String) {\n\tcreateRepository(name: $name, visibility: $visibility, description: $description, cloneUrl: $cloneUrl) {\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t\tid\n\t\tname\n\t}\n}\n") op.Var("name", name) op.Var("visibility", visibility) op.Var("description", description) diff --git a/srht/gitsrht/operations.graphql b/srht/gitsrht/operations.graphql index a2937a3..86eb27b 100644 --- a/srht/gitsrht/operations.graphql +++ b/srht/gitsrht/operations.graphql @@ -101,9 +101,13 @@ fragment repos on RepositoryCursor { name description visibility + readme owner { canonicalName } + HEAD { + name + } } cursor } @@ -213,6 +217,7 @@ mutation createRepository( owner { canonicalName } + id name } } diff --git a/srht/hgsrht/gql.go b/srht/hgsrht/gql.go index 99fa665..6eb8d13 100644 --- a/srht/hgsrht/gql.go +++ b/srht/hgsrht/gql.go @@ -419,7 +419,7 @@ func RepositoryIDByUser(client *gqlclient.Client, ctx context.Context, username } func Repositories(client *gqlclient.Client, ctx context.Context, cursor *Cursor) (repositories *RepositoryCursor, err error) { - op := gqlclient.NewOperation("query repositories ($cursor: Cursor) {\n\trepositories(cursor: $cursor) {\n\t\t... repos\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t}\n\tcursor\n}\n") + op := gqlclient.NewOperation("query repositories ($cursor: Cursor) {\n\trepositories(cursor: $cursor) {\n\t\t... repos\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\treadme\n\t\tnonPublishing\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t}\n\tcursor\n}\n") op.Var("cursor", cursor) var respData struct { Repositories *RepositoryCursor @@ -429,7 +429,7 @@ func Repositories(client *gqlclient.Client, ctx context.Context, cursor *Cursor) } func RepositoriesByUser(client *gqlclient.Client, ctx context.Context, username string, cursor *Cursor) (user *User, err error) { - op := gqlclient.NewOperation("query repositoriesByUser ($username: String!, $cursor: Cursor) {\n\tuser(username: $username) {\n\t\trepositories(cursor: $cursor) {\n\t\t\t... repos\n\t\t}\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t}\n\tcursor\n}\n") + op := gqlclient.NewOperation("query repositoriesByUser ($username: String!, $cursor: Cursor) {\n\tuser(username: $username) {\n\t\trepositories(cursor: $cursor) {\n\t\t\t... repos\n\t\t}\n\t}\n}\nfragment repos on RepositoryCursor {\n\tresults {\n\t\tname\n\t\tdescription\n\t\tvisibility\n\t\treadme\n\t\tnonPublishing\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t}\n\tcursor\n}\n") op.Var("username", username) op.Var("cursor", cursor) var respData struct { @@ -450,7 +450,7 @@ func UserWebhooks(client *gqlclient.Client, ctx context.Context, cursor *Cursor) } func CreateRepository(client *gqlclient.Client, ctx context.Context, name string, visibility Visibility, description string) (createRepository *Repository, err error) { - op := gqlclient.NewOperation("mutation createRepository ($name: String!, $visibility: Visibility!, $description: String!) {\n\tcreateRepository(name: $name, visibility: $visibility, description: $description) {\n\t\tname\n\t}\n}\n") + op := gqlclient.NewOperation("mutation createRepository ($name: String!, $visibility: Visibility!, $description: String!) {\n\tcreateRepository(name: $name, visibility: $visibility, description: $description) {\n\t\tid\n\t\tname\n\t\towner {\n\t\t\tcanonicalName\n\t\t}\n\t}\n}\n") op.Var("name", name) op.Var("visibility", visibility) op.Var("description", description) @@ -461,6 +461,17 @@ func CreateRepository(client *gqlclient.Client, ctx context.Context, name string return respData.CreateRepository, err } +func UpdateRepository(client *gqlclient.Client, ctx context.Context, id int32, input RepoInput) (updateRepository *Repository, err error) { + op := gqlclient.NewOperation("mutation updateRepository ($id: Int!, $input: RepoInput!) {\n\tupdateRepository(id: $id, input: $input) {\n\t\tid\n\t}\n}\n") + op.Var("id", id) + op.Var("input", input) + var respData struct { + UpdateRepository *Repository + } + err = client.Execute(ctx, op, &respData) + return respData.UpdateRepository, err +} + func DeleteRepository(client *gqlclient.Client, ctx context.Context, id int32) (deleteRepository *Repository, err error) { op := gqlclient.NewOperation("mutation deleteRepository ($id: Int!) {\n\tdeleteRepository(id: $id) {\n\t\tname\n\t}\n}\n") op.Var("id", id) diff --git a/srht/hgsrht/operations.graphql b/srht/hgsrht/operations.graphql index f184da1..0eabb5f 100644 --- a/srht/hgsrht/operations.graphql +++ b/srht/hgsrht/operations.graphql @@ -33,6 +33,8 @@ fragment repos on RepositoryCursor { name description visibility + readme + nonPublishing owner { canonicalName } @@ -60,7 +62,23 @@ mutation createRepository( visibility: $visibility description: $description ) { + id name + owner { + canonicalName + } + } +} + +mutation updateRepository( + $id: Int! + $input: RepoInput! +) { + updateRepository( + id: $id + input: $input + ) { + id } } diff --git a/srht/listssrht/gql.go b/srht/listssrht/gql.go index 615cb13..b0ec0a0 100644 --- a/srht/listssrht/gql.go +++ b/srht/listssrht/gql.go @@ -728,7 +728,7 @@ func MailingLists(client *gqlclient.Client, ctx context.Context, cursor *Cursor) } func ExportMailingLists(client *gqlclient.Client, ctx context.Context, cursor *Cursor) (me *User, err error) { - op := gqlclient.NewOperation("query exportMailingLists ($cursor: Cursor) {\n\tme {\n\t\tlists(cursor: $cursor) {\n\t\t\tresults {\n\t\t\t\tname\n\t\t\t\tdescription\n\t\t\t\tpermitMime\n\t\t\t\trejectMime\n\t\t\t\tarchive\n\t\t\t}\n\t\t\tcursor\n\t\t}\n\t}\n}\n") + op := gqlclient.NewOperation("query exportMailingLists ($cursor: Cursor) {\n\tme {\n\t\tlists(cursor: $cursor) {\n\t\t\tresults {\n\t\t\t\tname\n\t\t\t\tdescription\n\t\t\t\tvisibility\n\t\t\t\tpermitMime\n\t\t\t\trejectMime\n\t\t\t\tarchive\n\t\t\t}\n\t\t\tcursor\n\t\t}\n\t}\n}\n") op.Var("cursor", cursor) var respData struct { Me *User @@ -983,7 +983,7 @@ func DeleteACL(client *gqlclient.Client, ctx context.Context, id int32) (deleteA } func CreateMailingList(client *gqlclient.Client, ctx context.Context, name string, description *string, visibility Visibility) (createMailingList *MailingList, err error) { - op := gqlclient.NewOperation("mutation createMailingList ($name: String!, $description: String, $visibility: Visibility!) {\n\tcreateMailingList(name: $name, description: $description, visibility: $visibility) {\n\t\tname\n\t}\n}\n") + op := gqlclient.NewOperation("mutation createMailingList ($name: String!, $description: String, $visibility: Visibility!) {\n\tcreateMailingList(name: $name, description: $description, visibility: $visibility) {\n\t\tname\n\t\tid\n\t}\n}\n") op.Var("name", name) op.Var("description", description) op.Var("visibility", visibility) @@ -994,6 +994,28 @@ func CreateMailingList(client *gqlclient.Client, ctx context.Context, name strin return respData.CreateMailingList, err } +func UpdateMailingList(client *gqlclient.Client, ctx context.Context, id int32, input MailingListInput) (updateMailingList *MailingList, err error) { + op := gqlclient.NewOperation("mutation updateMailingList ($id: Int!, $input: MailingListInput!) {\n\tupdateMailingList(id: $id, input: $input) {\n\t\tid\n\t}\n}\n") + op.Var("id", id) + op.Var("input", input) + var respData struct { + UpdateMailingList *MailingList + } + err = client.Execute(ctx, op, &respData) + return respData.UpdateMailingList, err +} + +func ImportMailingListSpool(client *gqlclient.Client, ctx context.Context, listID int32, spool gqlclient.Upload) (importMailingListSpool bool, err error) { + op := gqlclient.NewOperation("mutation importMailingListSpool ($listID: Int!, $spool: Upload!) {\n\timportMailingListSpool(listID: $listID, spool: $spool)\n}\n") + op.Var("listID", listID) + op.Var("spool", spool) + var respData struct { + ImportMailingListSpool bool + } + err = client.Execute(ctx, op, &respData) + return respData.ImportMailingListSpool, err +} + func CreateUserWebhook(client *gqlclient.Client, ctx context.Context, config UserWebhookInput) (createUserWebhook *WebhookSubscription, err error) { op := gqlclient.NewOperation("mutation createUserWebhook ($config: UserWebhookInput!) {\n\tcreateUserWebhook(config: $config) {\n\t\tid\n\t}\n}\n") op.Var("config", config) diff --git a/srht/listssrht/operations.graphql b/srht/listssrht/operations.graphql index d3c46d9..3629f96 100644 --- a/srht/listssrht/operations.graphql +++ b/srht/listssrht/operations.graphql @@ -18,6 +18,7 @@ query exportMailingLists($cursor: Cursor) { results { name description + visibility permitMime rejectMime archive @@ -341,9 +342,26 @@ mutation createMailingList( visibility: $visibility ) { name + id } } +mutation updateMailingList( + $id: Int! + $input: MailingListInput! +) { + updateMailingList( + id: $id + input: $input + ) { + id + } +} + +mutation importMailingListSpool($listID: Int!, $spool: Upload!) { + importMailingListSpool(listID: $listID, spool: $spool) +} + mutation createUserWebhook($config: UserWebhookInput!) { createUserWebhook(config: $config) { id diff --git a/srht/todosrht/gql.go b/srht/todosrht/gql.go index ac6b2aa..4626dc6 100644 --- a/srht/todosrht/gql.go +++ b/srht/todosrht/gql.go @@ -1385,7 +1385,7 @@ func UnassignUser(client *gqlclient.Client, ctx context.Context, trackerId int32 } func CreateTracker(client *gqlclient.Client, ctx context.Context, name string, description *string, visibility Visibility) (createTracker *Tracker, err error) { - op := gqlclient.NewOperation("mutation createTracker ($name: String!, $description: String, $visibility: Visibility!) {\n\tcreateTracker(name: $name, description: $description, visibility: $visibility) {\n\t\tname\n\t}\n}\n") + op := gqlclient.NewOperation("mutation createTracker ($name: String!, $description: String, $visibility: Visibility!) {\n\tcreateTracker(name: $name, description: $description, visibility: $visibility) {\n\t\tname\n\t\tid\n\t}\n}\n") op.Var("name", name) op.Var("description", description) op.Var("visibility", visibility) @@ -1396,6 +1396,17 @@ func CreateTracker(client *gqlclient.Client, ctx context.Context, name string, d return respData.CreateTracker, err } +func ImportTrackerDump(client *gqlclient.Client, ctx context.Context, trackerId int32, dump gqlclient.Upload) (importTrackerDump bool, err error) { + op := gqlclient.NewOperation("mutation importTrackerDump ($trackerId: Int!, $dump: Upload!) {\n\timportTrackerDump(trackerId: $trackerId, dump: $dump)\n}\n") + op.Var("trackerId", trackerId) + op.Var("dump", dump) + var respData struct { + ImportTrackerDump bool + } + err = client.Execute(ctx, op, &respData) + return respData.ImportTrackerDump, err +} + func DeleteTicket(client *gqlclient.Client, ctx context.Context, trackerId int32, ticketId int32) (deleteTicket *Ticket, err error) { op := gqlclient.NewOperation("mutation deleteTicket ($trackerId: Int!, $ticketId: Int!) {\n\tdeleteTicket(trackerId: $trackerId, ticketId: $ticketId) {\n\t\tsubject\n\t}\n}\n") op.Var("trackerId", trackerId) diff --git a/srht/todosrht/operations.graphql b/srht/todosrht/operations.graphql index a9d0c2f..2a9ff26 100644 --- a/srht/todosrht/operations.graphql +++ b/srht/todosrht/operations.graphql @@ -589,9 +589,14 @@ mutation createTracker( visibility: $visibility ) { name + id } } +mutation importTrackerDump($trackerId: Int!, $dump: Upload!) { + importTrackerDump(trackerId: $trackerId, dump: $dump) +} + mutation deleteTicket($trackerId: Int!, $ticketId: Int!) { deleteTicket(trackerId: $trackerId, ticketId: $ticketId) { subject base-commit: 02bae6b3ac443125aeb7e1227905ee081e995b8e -- 2.41.0
Sorry for the delay, I filed this patch for later and completely forgot about it. It would be nice if you could rebase it, since Simon already picked some changes and applied them in fb9c7be9970c80c35402874c09c106241cb694b7. To test it, I applied this patch on top of 02bae6b3ac443125aeb7e1227905ee081e995b8e.