This adds the helper script contrib/unmigrate-logs, which complements
contrib/migrate-logs to offer round-trip mobility of database history to and
from msgstore 'fs' logs. It can run as a cron job to update an ancillary store
for things like maintaining a centralized file-based search index.
---
contrib/unmigrate-logs/main.go | 388 ++++++++++++++++++
contrib/unmigrate-logs/main_test.go | 362 ++++++++++++++++
.../soju-test-user/testnet/#a/2024-06-04.log | 5 +
.../soju-test-user/testnet/#a/2024-06-05.log | 1 +
.../soju-test-user/testnet/#b/2024-06-04.log | 1 +
.../soju-test-user/testnet/#b/2024-06-05.log | 1 +
.../testnet/memoserv/2024-06-04.log | 2 +
.../testnet/tester/2024-06-04.log | 2 +
.../testnet/tester/2024-06-05.log | 2 +
.../soju-test-user/testnet/#a/2024-06-04.log | 5 +
.../soju-test-user/testnet/#a/2024-06-05.log | 2 +
.../soju-test-user/testnet/#b/2024-06-04.log | 1 +
.../soju-test-user/testnet/#b/2024-06-05.log | 2 +
.../testnet/memoserv/2024-06-04.log | 2 +
.../testnet/memoserv/2024-06-05.log | 1 +
.../testnet/tester/2024-06-04.log | 2 +
.../testnet/tester/2024-06-05.log | 4 +
.../soju-test-user/testnet/#a/2024-06-04.log | 5 +
.../soju-test-user/testnet/#a/2024-06-05.log | 3 +
.../soju-test-user/testnet/#b/2024-06-04.log | 1 +
.../soju-test-user/testnet/#b/2024-06-05.log | 3 +
.../testnet/memoserv/2024-06-04.log | 2 +
.../testnet/memoserv/2024-06-05.log | 1 +
.../testnet/tester/2024-06-04.log | 2 +
.../testnet/tester/2024-06-05.log | 6 +
.../soju-test-user/testnet/#a/2024-06-04.log | 5 +
.../soju-test-user/testnet/#b/2024-06-04.log | 1 +
.../testnet/memoserv/2024-06-04.log | 2 +
.../testnet/tester/2024-06-04.log | 2 +
.../soju-test-user/testnet/#a/2024-06-04.log | 5 +
.../soju-test-user/testnet/#a/2024-06-05.log | 3 +
.../soju-test-user/testnet/#b/2024-06-04.log | 1 +
.../soju-test-user/testnet/#b/2024-06-05.log | 3 +
.../testnet/memoserv/2024-06-04.log | 2 +
.../testnet/memoserv/2024-06-05.log | 1 +
.../testnet/tester/2024-06-04.log | 2 +
.../testnet/tester/2024-06-05.log | 6 +
contrib/unmigrate-logs/testdata/raw_align.log | 3 +
.../unmigrate-logs/testdata/raw_chan_a.log | 15 +
.../unmigrate-logs/testdata/raw_chan_b.log | 11 +
.../unmigrate-logs/testdata/raw_query_a.log | 8 +
.../unmigrate-logs/testdata/raw_query_b.log | 16 +
42 files changed, 892 insertions(+)
create mode 100644 contrib/unmigrate-logs/main.go
create mode 100644 contrib/unmigrate-logs/main_test.go
create mode 100644 contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/memoserv/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#a/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#b/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/memoserv/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/tester/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-04.log
create mode 100644 contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-05.log
create mode 100644 contrib/unmigrate-logs/testdata/raw_align.log
create mode 100644 contrib/unmigrate-logs/testdata/raw_chan_a.log
create mode 100644 contrib/unmigrate-logs/testdata/raw_chan_b.log
create mode 100644 contrib/unmigrate-logs/testdata/raw_query_a.log
create mode 100644 contrib/unmigrate-logs/testdata/raw_query_b.log
diff --git a/contrib/unmigrate-logs/main.go b/contrib/unmigrate-logs/main.go
new file mode 100644
index 0000000..ffe0ce9
--- /dev/null
+++ b/contrib/unmigrate-logs/main.go
@@ -0,0 +1,388 @@
+// Command unmigrate-logs exports messages from a database to a file tree.
+//
+// This script may need updating if action is ever taken on the following
+// issues from the tracker, which are all prefixed with "msgstorefs: ":
+//
+// - https://todo.sr.ht/~emersion/soju/53 "configurable logs timezone"
+// - https://todo.sr.ht/~emersion/soju/151 "use millisecond timestamp"
+// - https://todo.sr.ht/~emersion/soju/208 "improve filename escaping"
+package main
+
+import (
+ "bufio"
+ "context"
+ "flag"
+ "fmt"
+ "log"
+ "math"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "git.sr.ht/~emersion/soju/database"
+ "git.sr.ht/~emersion/soju/msgstore"
+ "git.sr.ht/~emersion/soju/msgstore/znclog"
+ "git.sr.ht/~emersion/soju/xirc"
+ "gopkg.in/irc.v4"
+)
+
+const usage = `usage: unmigrate-logs <database> <log-root>
+
+Export a Soju database to a new or existing log tree laid out like so:
+
+ <log-root>/{user}/{network}/{target}/{YYYY-mm-dd}.log
+
+This operation is lossy. Details like message tags and fractional timestamps
+do not survive conversion. This operation is also destructive on the
+destination side unless the -align option is given. All dumped messages are
+retained on the source/database side. Timestamps and log rotation reflect the
+current locale's timezone unless the environment variable TZ is present.
+
+<database> should be specified as "{driver}:{source}", where driver is
+'sqlite3' or 'postgres' and source is a spec supported the db entry in a Soju
+config. For example, a 'postgres' source has space-separated key/value pairs
+in the form of a lib/pq "connection string," like 'host=postgres.svc
+dbname=soju sslmode=disable'.
+
+<log-root> should be the filesystem location of the log tree's root directory.
+
+Options:
+
+ -help Show this help message
+
+ -align Only write whole days, beginning the day after each target's newest
+ existing log file and ending (and excluding) the day of the newest
+ database record for each network
+`
+
+func init() {
+ flag.Usage = func() {
+ fmt.Fprint(flag.CommandLine.Output(), usage)
+ }
+}
+
+// findLastFilename returns the name of the last file in a directory, if any.
+func findLastFilename(dir string) (string, error) {
+ files, err := os.ReadDir(dir)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return "", nil
+ }
+ return "", err
+ }
+ if len(files) == 0 {
+ return "", nil
+ }
+
+ return files[len(files)-1].Name(), nil
+}
+
+// readLastLines gathers lines sharing a time stamp in a log's tail.
+func readLastLines(filePath string) ([]string, error) {
+ file, err := os.Open(filePath)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ var matches []string
+ var current string
+ timestampPrefixLen := len("[01:02:03] ")
+ scanner := bufio.NewScanner(file)
+
+ // This may be wasteful, but scanning backwards seems complicated.
+ for scanner.Scan() {
+ line := scanner.Text()
+ if len(line) <= timestampPrefixLen {
+ continue
+ }
+ if line[:timestampPrefixLen] == current {
+ matches = append(matches, line)
+ continue
+ }
+ current = line[:timestampPrefixLen]
+ matches = []string{line}
+ }
+
+ return matches, scanner.Err()
+}
+
+// generateCompositeKey creates a map key for deduping.
+func generateCompositeKey(
+ msg *irc.Message,
+ user *database.User,
+ network *database.Network,
+ target string,
+) string {
+ params := append([]string(nil), msg.Params...)
+ switch msg.Command {
+ // Ignore trailing garbage after a JOIN command's target param.
+ case "JOIN":
+ params = params[:1]
+ case "PRIVMSG", "NOTICE":
+ // For queries, replace the target parameter with our own nick,
+ // mimicking msgstore's UnmarshalLine(). Targets as first parameters
+ // aren't normally casemapped when they arrive from the server, but
+ // these params were reconstituted from FS logs and stored targets.
+ // And since we're offline and casemapping is per-server, we can only
+ // use the default mapping.
+ if xirc.CaseMappingRFC1459(msg.Name) == target {
+ params[0] = database.GetNick(user, network)
+ }
+ params[0] = xirc.CaseMappingRFC1459(params[0])
+ }
+ return fmt.Sprintf("%s|%s|%v", msg.Name, msg.Command, params)
+}
+
+// extractDateRef generates a date from a ZNC-style log's file name.
+func extractDateRef(logFile string) (time.Time, error) {
+ var year, month, day int
+ _, err := fmt.Sscanf(logFile, "%04d-%02d-%02d.log", &year, &month, &day)
+ if err != nil {
+ return time.Time{}, fmt.Errorf(
+ "failed scanning filename %s: %w", logFile, err,
+ )
+ }
+ return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local), nil
+}
+
+// createDeduper returns a predicate filter for deduping boundary messages.
+//
+// The deduplication strategy depends on database.Database.ListMessages()
+// preserving a consistent ordering across sessions for messages sharing a
+// time stamp.
+func createDeduper(
+ user *database.User,
+ network *database.Network,
+ target, logFile, dir string,
+ opts *database.MessageOptions,
+) (func(*irc.Message) bool, error) {
+
+ dateRef, err := extractDateRef(logFile)
+ if err != nil {
+ return nil, fmt.Errorf("failed to extract date ref: %w", err)
+ }
+
+ // Track false positives. For example, a query target for the Soju user's
+ // own nick can have two identical messages per timestamp, or a services
+ // bot's help command can emit multiple empty lines in a logical response.
+ seen := make(map[string]int)
+ var lastTime time.Time
+ if opts.BeforeTime.IsZero() {
+ lastLines, err := readLastLines(filepath.Join(dir, logFile))
+ if err != nil {
+ return nil, fmt.Errorf("failed to read last lines: %w", err)
+ }
+ lastTime = dateRef // in case lastLines is empty
+ for _, lastLine := range lastLines {
+ msg, ts, err := znclog.UnmarshalLine(
+ lastLine, user, network, target, dateRef, true,
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to unmarshall line: %w", err)
+ }
+ lastTime = ts
+ seen[generateCompositeKey(msg, user, network, target)] += 1
+ }
+ } else { // option -align given, so begin the day after dateRef
+ lastTime = dateRef.AddDate(0, 0, 1)
+ }
+
+ log.Printf(" Beg: %s", lastTime)
+ timeKey := xirc.FormatServerTime(lastTime)
+ // Disregard trailing subsecs portion because ZNC-style logs contain
+ // degenerate stamps lacking fractional secs.
+ timeKey = timeKey[:len(timeKey)-len(".000Z")]
+ // Ensure messages at the boundary are considered.
+ opts.AfterTime = lastTime.Add(-time.Nanosecond)
+
+ return func(message *irc.Message) bool {
+ if len(seen) != 0 && timeKey == message.Tags["time"][:len(timeKey)] {
+ key := generateCompositeKey(message, user, network, target)
+ if _, found := seen[key]; found {
+ seen[key] -= 1
+ if seen[key] == 0 {
+ delete(seen, key)
+ }
+ log.Printf(" Dup: %s from %s", message.Command, message.Name)
+ return true
+ }
+ }
+ return false
+ }, nil
+}
+
+// processTarget saves a user's database messages for one casemapped target.
+func processTarget(
+ ctx context.Context,
+ db database.Database,
+ fs msgstore.Store,
+ user *database.User,
+ network *database.Network,
+ target, logRoot string,
+ opts database.MessageOptions, // new copy
+) error {
+
+ dir := filepath.Join(
+ logRoot,
+ msgstore.EscapeFilename(user.Username),
+ msgstore.EscapeFilename(network.GetName()),
+ msgstore.EscapeFilename(target),
+ )
+ logFile, err := findLastFilename(dir)
+ if err != nil {
+ return fmt.Errorf("failed to find last log file: %w", err)
+ }
+
+ var isDuplicate func(*irc.Message) bool
+ if logFile != "" {
+ isDuplicate, err = createDeduper(
+ user, network, target, logFile, dir, &opts,
+ )
+ }
+ if err != nil {
+ return fmt.Errorf("failed to create deduplication filter: %w", err)
+ }
+ if !opts.BeforeTime.IsZero() {
+ log.Printf(" End: %s", opts.BeforeTime)
+ if !opts.AfterTime.Before(opts.BeforeTime) {
+ // This can happen, e.g., if records disappear from the database
+ // between runs or the user switches between -align and non, etc.
+ return nil
+ }
+ }
+
+ messages, err := db.ListMessages(ctx, network.ID, target, &opts)
+ if err != nil {
+ return fmt.Errorf("failed to get msg for target %s: %w", target, err)
+ }
+ for _, message := range messages {
+ if isDuplicate != nil && isDuplicate(message) {
+ continue
+ }
+ id, err := fs.Append(network, target, message)
+ if err != nil {
+ return fmt.Errorf("failed to append message %s: %w", id, err)
+ }
+ }
+
+ return nil
+}
+
+// processNetwork exports all messages for a user from a single network.
+func processNetwork(
+ ctx context.Context,
+ db database.Database,
+ fs msgstore.Store,
+ user *database.User,
+ network *database.Network,
+ logRoot string,
+ optAlign bool,
+) error {
+ opts := &database.MessageOptions{ // need Stable for ListMessages()
+ Limit: math.MaxInt, Events: true, Stable: true,
+ }
+ targets, err := db.ListMessageLastPerTarget(ctx, network.ID, opts)
+ if err != nil {
+ return fmt.Errorf(
+ "failed to get targets for network %s: %w", network.Name, err,
+ )
+ }
+
+ // Find the most recent message for any target across the network.
+ if optAlign {
+ t := time.Time{}
+ for _, target := range targets {
+ if target.LatestMessage.After(t) {
+ t = target.LatestMessage
+ }
+ }
+ opts.BeforeTime = time.Date(
+ t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.Local,
+ )
+ }
+
+ for _, target := range targets {
+ log.Printf(" Target: %s", target.Name)
+ if err := processTarget(
+ ctx, db, fs, user, network, target.Name, logRoot, *opts,
+ ); err != nil {
+ return fmt.Errorf(
+ "failed to save log for target %s: %w", target.Name, err,
+ )
+ }
+ }
+
+ return nil
+}
+
+// processUser exports all messages for a user to an FS msgstore.
+func processUser(
+ ctx context.Context,
+ db database.Database,
+ user *database.User,
+ logRoot string,
+ optAlign bool,
+) error {
+ log.Printf("User: %s", user.Username)
+
+ msgStore := msgstore.NewFSStore(logRoot, user)
+ networks, err := db.ListNetworks(ctx, user.ID)
+ if err != nil {
+ return fmt.Errorf(
+ "failed to get networks for user: %s(#%d): %w",
+ user.Username, user.ID, err,
+ )
+ }
+
+ for _, network := range networks {
+ log.Printf(" Network: %s", network.GetName())
+ if err := processNetwork(
+ ctx, db, msgStore, user, &network, logRoot, optAlign,
+ ); err != nil {
+ return fmt.Errorf(
+ "failed to save log for network %s: %w",
+ network.GetName(), err,
+ )
+ }
+ }
+ return msgStore.Close()
+}
+
+func main() {
+ alignFlag := flag.Bool("align", false, "Align to whole days")
+ flag.Parse()
+ logRoot := flag.Arg(1)
+
+ dbParams := strings.SplitN(flag.Arg(0), ":", 2)
+ if len(dbParams) != 2 {
+ log.Fatalf("database not properly specified: %v", dbParams)
+ }
+ db, err := database.Open(dbParams[0], dbParams[1])
+ if err != nil {
+ log.Fatalf("failed to open database: %v", err)
+ }
+ defer db.Close()
+
+ // We could arrange to trap some signals here and restore backups of
+ // pre-modified files. OTOH, it *should* be common knowledge that
+ // artificially terminating this process can corrupt existing logs, right?
+ ctx := context.Background()
+
+ users, err := db.ListUsers(ctx)
+ if err != nil {
+ log.Fatalf("unable to get users: %v", err)
+ }
+
+ if *alignFlag {
+ log.Println("Option align present: limiting new logs to whole days")
+ }
+
+ for _, user := range users {
+ err = processUser(ctx, db, &user, logRoot, *alignFlag)
+ if err != nil {
+ log.Fatalf("failed to process user %s: %v", user.Nick, err)
+ }
+ }
+}
diff --git a/contrib/unmigrate-logs/main_test.go b/contrib/unmigrate-logs/main_test.go
new file mode 100644
index 0000000..69bc481
--- /dev/null
+++ b/contrib/unmigrate-logs/main_test.go
@@ -0,0 +1,362 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ _ "embed"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "git.sr.ht/~emersion/soju/database"
+ "gopkg.in/irc.v4"
+)
+
+const (
+ testUsername = "soju-test-user"
+ testPassword = testUsername
+)
+
+//go:embed testdata/raw_chan_a.log
+var rawChanLogA string
+var chanStepsA [][]*irc.Message
+
+//go:embed testdata/raw_chan_b.log
+var rawChanLogB string
+var chanStepsB [][]*irc.Message
+
+//go:embed testdata/raw_query_a.log
+var rawQueryLogA string
+var queryStepsA [][]*irc.Message
+
+//go:embed testdata/raw_query_b.log
+var rawQueryLogB string
+var queryStepsB [][]*irc.Message
+
+//go:embed testdata/raw_align.log
+var rawAlignLog string
+var alignSteps [][]*irc.Message
+
+// bundleSteps partitions raw logs into sequences of simulation steps.
+func bundleSteps(t *testing.T, messages string) [][]*irc.Message {
+ var all [][]*irc.Message
+ var msgs []*irc.Message
+
+ for _, line := range strings.Split(messages, "\n") {
+ // Skip blanks and comments.
+ if line == "" {
+ continue
+ }
+ // Commit hunks terminated by a line matching ^# %[0-9]$.
+ if line[0:1] == "#" {
+ if len(line) > 2 && line[0:3] == "# %" {
+ all = append(all, msgs)
+ msgs = []*irc.Message{}
+ }
+ continue
+ }
+
+ msg, err := irc.ParseMessage(line)
+ if err != nil {
+ t.Fatalf("failed to parse message: %v", err)
+ }
+ msgs = append(msgs, msg)
+ }
+ if len(msgs) != 0 {
+ all = append(all, msgs)
+ }
+ return all
+}
+
+func createTempSqliteDB(t *testing.T) database.Database {
+ if !database.SqliteEnabled {
+ t.Skip("SQLite support is disabled")
+ }
+
+ db, err := database.OpenTempSqliteDB()
+ if err != nil {
+ t.Fatalf("failed to create temporary SQLite database: %v", err)
+ }
+ return db
+}
+
+func createTempPostgresDB(t *testing.T) database.Database {
+ source, ok := os.LookupEnv("SOJU_TEST_POSTGRES")
+ // This wants a lib/pq spec, like
+ // host=127.0.0.1 user=postgres password=postgres sslmode=disable
+ // Not a URL.
+ if !ok {
+ t.Skip("set SOJU_TEST_POSTGRES to a connection string to execute PostgreSQL tests")
+ }
+
+ db, err := database.OpenTempPostgresDB(source)
+ if err != nil {
+ t.Fatalf("failed to create temporary PostgreSQL database: %v", err)
+ }
+
+ return db
+}
+
+func createTestUser(t *testing.T, db database.Database) *database.User {
+ record := database.NewUser(testUsername)
+ if err := record.SetPassword(testPassword); err != nil {
+ t.Fatalf("failed to generate bcrypt hash: %v", err)
+ }
+ if err := db.StoreUser(context.Background(), record); err != nil {
+ t.Fatalf("failed to store test user: %v", err)
+ }
+
+ return record
+}
+
+func createTestNetwork(
+ t *testing.T, db database.Database, user *database.User,
+) *database.Network {
+
+ network := database.NewNetwork("ircs://irc.testnet.fake")
+ network.Name = "testnet"
+ // For these fake sessions, our network user has the nickname "tester",
+ // which is also the network account name but not the Soju account name.
+ network.Nick = "tester"
+
+ if err := db.StoreNetwork(context.Background(), user.ID, network); err != nil {
+ t.Fatalf("failed to store test network: %v", err)
+ }
+
+ return network
+}
+
+func copyFile(src, dst string) error {
+ in, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+
+ out, err := os.Create(dst)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+
+ if _, err := io.Copy(out, in); err != nil {
+ return err
+ }
+
+ return out.Close()
+}
+
+// copyDir recursively copies directory src to dest.
+func copyDir(src, dst string) error {
+ entries, err := os.ReadDir(src)
+ if err != nil {
+ return err
+ }
+
+ if err := os.MkdirAll(dst, os.ModePerm); err != nil {
+ return err
+ }
+
+ for _, entry := range entries {
+ srcPath := filepath.Join(src, entry.Name())
+ dstPath := filepath.Join(dst, entry.Name())
+
+ if entry.IsDir() {
+ if err := copyDir(srcPath, dstPath); err != nil {
+ return err
+ }
+ } else {
+ if err := copyFile(srcPath, dstPath); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+// populateStep recursively copies a subtree from testdata to logRoot.
+func populateStep(t *testing.T, logRoot, expectTree string) {
+ src := filepath.Join("testdata", expectTree)
+ if err := copyDir(src, logRoot); err != nil {
+ t.Fatalf("Failed to prepopulate step %s: %v", expectTree, err)
+ }
+}
+
+// loadMessages inserts irc.Message messages into a database.
+func loadMessages(
+ t *testing.T,
+ network *database.Network,
+ db database.Database,
+ entity string,
+ msgs ...[]*irc.Message,
+) {
+ var messages []*irc.Message
+ for _, msg := range msgs {
+ messages = append(messages, msg...)
+ }
+ _, err := db.StoreMessages(context.TODO(), network.ID, entity, messages)
+ if err != nil {
+ t.Fatalf("failed to append message: %v", err)
+ }
+}
+
+// runWriteUser simulates main() without the flag parsing.
+func runWriteUser(
+ t *testing.T,
+ db database.Database,
+ user *database.User,
+ logRoot string,
+ optAlign bool,
+) {
+ // Make the script thinks it's 12AM on 2024-06-06.
+ if err := processUser(
+ context.TODO(), db, user, logRoot, optAlign,
+ ); err != nil {
+ t.Fatalf("failed to write users: %v", err)
+ }
+}
+
+func compareDirs(
+ t *testing.T, dirA, dirB, pathA string, infoA os.FileInfo, errA error,
+) (string, string, os.FileInfo) {
+ if errA != nil {
+ t.Fatalf("failed accessing path %s: %v", pathA, errA)
+ }
+
+ relPath, err := filepath.Rel(dirA, pathA)
+ if err != nil {
+ t.Fatalf("failed finding relative path: %v", err)
+ }
+
+ pathB := filepath.Join(dirB, relPath)
+ infoB, errB := os.Stat(pathB)
+
+ if os.IsNotExist(errB) {
+ t.Fatalf("mismatch: %s exists but %s does not", pathA, pathB)
+
+ } else if errB != nil {
+ t.Fatalf("failed accessing %s: %v", pathB, errB)
+
+ } else if infoA.IsDir() != infoB.IsDir() {
+ if infoA.IsDir() {
+ t.Fatalf("mismatch: %s is a directory %s is not", relPath, pathB)
+ }
+ t.Fatalf("mismatch: %s is a directory %s is not", pathB, relPath)
+ }
+
+ return relPath, pathB, infoB
+}
+
+// assertDirs asserts directories A and B have the same structure and content.
+// It's akin to running: diff -r A B.
+func assertDirs(t *testing.T, logRoot, expectDir string) {
+ dirA := logRoot
+ dirB := filepath.Join("testdata", expectDir)
+
+ filepath.Walk(dirA, func(pathA string, infoA os.FileInfo, errA error) error {
+
+ // Assert {paths in A} - {paths in B} is null.
+ relPath, pathB, infoB := compareDirs(t, dirA, dirB, pathA, infoA, errA)
+
+ // Assert contents of common files match.
+ if !infoA.IsDir() {
+ contentA, err := os.ReadFile(pathA)
+ if err != nil {
+ t.Fatalf("failed reading file %s: %v", pathA, err)
+ }
+ contentB, err := os.ReadFile(pathB)
+ if err != nil {
+ t.Fatalf("failed reading file %s: %v", pathB, err)
+ }
+ if !bytes.Equal(contentA, contentB) {
+ t.Fatalf(
+ "mismatch at %s\n\n%s (%dB):\n%s\n%s (%dB):\n%s\n",
+ relPath,
+ pathA, infoA.Size(), contentA,
+ pathB, infoB.Size(), contentB,
+ )
+ }
+ }
+ return nil
+ })
+
+ filepath.Walk(dirB, func(pathB string, infoB os.FileInfo, errB error) error {
+
+ // Assert B - A is also null.
+ compareDirs(t, dirB, dirA, pathB, infoB, errB)
+ return nil
+ })
+}
+
+func testOutput(t *testing.T, db database.Database) {
+ originalLocal := time.Local
+ defer func() { time.Local = originalLocal }()
+ time.Local = time.UTC
+
+ var logRoot string
+ user := createTestUser(t, db)
+ network := createTestNetwork(t, db, user)
+
+ // #a: QUIT, #b: NICK, memoserv: PRIVMSG/NOTICE
+ logRoot = t.TempDir()
+ populateStep(t, logRoot, "expect_1")
+ loadMessages(t, network, db, "#a", chanStepsA[:1]...)
+ loadMessages(t, network, db, "#b", chanStepsB[:1]...)
+ loadMessages(t, network, db, "tester", queryStepsB[:1]...)
+ loadMessages(t, network, db, "memoserv", queryStepsA[:1]...)
+ runWriteUser(t, db, user, logRoot, false)
+ assertDirs(t, logRoot, "expect_1")
+ runWriteUser(t, db, user, logRoot, false) // x2 ~~> idempotent
+ assertDirs(t, logRoot, "expect_1")
+
+ // #a: JOIN, #b: TOPIC, memoserv: NOTICE
+ logRoot = t.TempDir()
+ loadMessages(t, network, db, "#a", chanStepsA[1])
+ loadMessages(t, network, db, "#b", chanStepsB[1])
+ loadMessages(t, network, db, "tester", queryStepsB[1])
+ loadMessages(t, network, db, "memoserv", queryStepsA[1])
+ runWriteUser(t, db, user, logRoot, false)
+ assertDirs(t, logRoot, "expect_2")
+ runWriteUser(t, db, user, logRoot, false) // x2
+ assertDirs(t, logRoot, "expect_2")
+
+ // #a: PRIVMSG, #b: MODE
+ logRoot = t.TempDir()
+ loadMessages(t, network, db, "#a", chanStepsA[2])
+ loadMessages(t, network, db, "#b", chanStepsB[2])
+ loadMessages(t, network, db, "tester", queryStepsB[2])
+ runWriteUser(t, db, user, logRoot, false)
+ assertDirs(t, logRoot, "expect_3")
+ runWriteUser(t, db, user, logRoot, false) // x2
+ assertDirs(t, logRoot, "expect_3")
+
+ // Option: -align
+ logRoot = t.TempDir()
+ populateStep(t, logRoot, "expect_a1")
+ loadMessages(t, network, db, "#a", alignSteps...)
+ loadMessages(t, network, db, "#b", alignSteps...)
+ runWriteUser(t, db, user, logRoot, true)
+ assertDirs(t, logRoot, "expect_a2")
+ runWriteUser(t, db, user, logRoot, true) // x2
+ assertDirs(t, logRoot, "expect_a2")
+}
+
+func TestOutput(t *testing.T) {
+ chanStepsA = bundleSteps(t, rawChanLogA)
+ chanStepsB = bundleSteps(t, rawChanLogB)
+ queryStepsA = bundleSteps(t, rawQueryLogA)
+ queryStepsB = bundleSteps(t, rawQueryLogB)
+ alignSteps = bundleSteps(t, rawAlignLog)
+
+ t.Run("sqlite", func(t *testing.T) {
+ testOutput(t, createTempSqliteDB(t))
+ })
+
+ t.Run("postgres", func(t *testing.T) {
+ testOutput(t, createTempPostgresDB(t))
+ })
+}
diff --git a/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-04.log
new file mode 100644
index 0000000..1d447bd
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-04.log
@@ -0,0 +1,5 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
+[06:37:14] *** Joins: bob (~u@u/bob)
+[06:45:59] *** Joins: alice (~u@u/alice)
+[16:04:37] *** Quits: bob (~u@u/bob) (Remote host closed the connection)
+[16:13:29] *** Joins: bob (~u@u/bob)
diff --git a/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-05.log
new file mode 100644
index 0000000..bb517e8
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#a/2024-06-05.log
@@ -0,0 +1 @@
+[01:00:00] *** Quits: bob (~u@u/bob) (Ping timeout: 246 seconds)
diff --git a/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-04.log
new file mode 100644
index 0000000..82979bc
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-04.log
@@ -0,0 +1 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
diff --git a/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-05.log
new file mode 100644
index 0000000..0d1b8a0
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/#b/2024-06-05.log
@@ -0,0 +1 @@
+[01:00:00] *** dummy is now known as Guest1234
diff --git a/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/memoserv/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/memoserv/2024-06-04.log
new file mode 100644
index 0000000..fb96300
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/memoserv/2024-06-04.log
@@ -0,0 +1,2 @@
+[10:24:42] <tester> send alice Thanks.
+[10:24:42] -MemoServ- The memo has been successfully sent to alice.
diff --git a/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-04.log
new file mode 100644
index 0000000..1772e00
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-04.log
@@ -0,0 +1,2 @@
+[16:09:20] <tester> one
+[16:09:20] <tester> one
diff --git a/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-05.log
new file mode 100644
index 0000000..20c77c8
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_1/soju-test-user/testnet/tester/2024-06-05.log
@@ -0,0 +1,2 @@
+[01:00:00] <tester> two
+[01:00:00] <tester> two
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-04.log
new file mode 100644
index 0000000..1d447bd
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-04.log
@@ -0,0 +1,5 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
+[06:37:14] *** Joins: bob (~u@u/bob)
+[06:45:59] *** Joins: alice (~u@u/alice)
+[16:04:37] *** Quits: bob (~u@u/bob) (Remote host closed the connection)
+[16:13:29] *** Joins: bob (~u@u/bob)
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-05.log
new file mode 100644
index 0000000..70ac92c
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#a/2024-06-05.log
@@ -0,0 +1,2 @@
+[01:00:00] *** Quits: bob (~u@u/bob) (Ping timeout: 246 seconds)
+[02:00:00] *** Joins: bob (~u@u/bob)
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-04.log
new file mode 100644
index 0000000..82979bc
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-04.log
@@ -0,0 +1 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-05.log
new file mode 100644
index 0000000..0e5e43c
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/#b/2024-06-05.log
@@ -0,0 +1,2 @@
+[01:00:00] *** dummy is now known as Guest1234
+[02:00:00] *** bob changes topic to 'Hear ye...'
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-04.log
new file mode 100644
index 0000000..fb96300
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-04.log
@@ -0,0 +1,2 @@
+[10:24:42] <tester> send alice Thanks.
+[10:24:42] -MemoServ- The memo has been successfully sent to alice.
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-05.log
new file mode 100644
index 0000000..a7de83f
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/memoserv/2024-06-05.log
@@ -0,0 +1 @@
+[08:00:00] -MemoServ- alice has read your memo, which was sent at Jun 04 10:24:42 2024 +0000
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-04.log
new file mode 100644
index 0000000..1772e00
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-04.log
@@ -0,0 +1,2 @@
+[16:09:20] <tester> one
+[16:09:20] <tester> one
diff --git a/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-05.log
new file mode 100644
index 0000000..37d2f67
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_2/soju-test-user/testnet/tester/2024-06-05.log
@@ -0,0 +1,4 @@
+[01:00:00] <tester> two
+[01:00:00] <tester> two
+[02:00:00] <tester> three
+[02:00:00] <tester> three
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-04.log
new file mode 100644
index 0000000..1d447bd
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-04.log
@@ -0,0 +1,5 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
+[06:37:14] *** Joins: bob (~u@u/bob)
+[06:45:59] *** Joins: alice (~u@u/alice)
+[16:04:37] *** Quits: bob (~u@u/bob) (Remote host closed the connection)
+[16:13:29] *** Joins: bob (~u@u/bob)
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-05.log
new file mode 100644
index 0000000..26a7744
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#a/2024-06-05.log
@@ -0,0 +1,3 @@
+[01:00:00] *** Quits: bob (~u@u/bob) (Ping timeout: 246 seconds)
+[02:00:00] *** Joins: bob (~u@u/bob)
+[03:00:00] <alice> one
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-04.log
new file mode 100644
index 0000000..82979bc
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-04.log
@@ -0,0 +1 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-05.log
new file mode 100644
index 0000000..182b36b
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/#b/2024-06-05.log
@@ -0,0 +1,3 @@
+[01:00:00] *** dummy is now known as Guest1234
+[02:00:00] *** bob changes topic to 'Hear ye...'
+[03:00:00] *** ChanServ sets mode: +o bob
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-04.log
new file mode 100644
index 0000000..fb96300
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-04.log
@@ -0,0 +1,2 @@
+[10:24:42] <tester> send alice Thanks.
+[10:24:42] -MemoServ- The memo has been successfully sent to alice.
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-05.log
new file mode 100644
index 0000000..a7de83f
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/memoserv/2024-06-05.log
@@ -0,0 +1 @@
+[08:00:00] -MemoServ- alice has read your memo, which was sent at Jun 04 10:24:42 2024 +0000
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-04.log
new file mode 100644
index 0000000..1772e00
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-04.log
@@ -0,0 +1,2 @@
+[16:09:20] <tester> one
+[16:09:20] <tester> one
diff --git a/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-05.log
new file mode 100644
index 0000000..bff2ccb
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_3/soju-test-user/testnet/tester/2024-06-05.log
@@ -0,0 +1,6 @@
+[01:00:00] <tester> two
+[01:00:00] <tester> two
+[02:00:00] <tester> three
+[02:00:00] <tester> three
+[03:00:00] <tester> four
+[03:00:00] <tester> four
diff --git a/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#a/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#a/2024-06-04.log
new file mode 100644
index 0000000..1d447bd
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#a/2024-06-04.log
@@ -0,0 +1,5 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
+[06:37:14] *** Joins: bob (~u@u/bob)
+[06:45:59] *** Joins: alice (~u@u/alice)
+[16:04:37] *** Quits: bob (~u@u/bob) (Remote host closed the connection)
+[16:13:29] *** Joins: bob (~u@u/bob)
diff --git a/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#b/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#b/2024-06-04.log
new file mode 100644
index 0000000..82979bc
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/#b/2024-06-04.log
@@ -0,0 +1 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
diff --git a/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/memoserv/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/memoserv/2024-06-04.log
new file mode 100644
index 0000000..fb96300
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/memoserv/2024-06-04.log
@@ -0,0 +1,2 @@
+[10:24:42] <tester> send alice Thanks.
+[10:24:42] -MemoServ- The memo has been successfully sent to alice.
diff --git a/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/tester/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/tester/2024-06-04.log
new file mode 100644
index 0000000..1772e00
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a1/soju-test-user/testnet/tester/2024-06-04.log
@@ -0,0 +1,2 @@
+[16:09:20] <tester> one
+[16:09:20] <tester> one
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-04.log
new file mode 100644
index 0000000..1d447bd
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-04.log
@@ -0,0 +1,5 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
+[06:37:14] *** Joins: bob (~u@u/bob)
+[06:45:59] *** Joins: alice (~u@u/alice)
+[16:04:37] *** Quits: bob (~u@u/bob) (Remote host closed the connection)
+[16:13:29] *** Joins: bob (~u@u/bob)
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-05.log
new file mode 100644
index 0000000..26a7744
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#a/2024-06-05.log
@@ -0,0 +1,3 @@
+[01:00:00] *** Quits: bob (~u@u/bob) (Ping timeout: 246 seconds)
+[02:00:00] *** Joins: bob (~u@u/bob)
+[03:00:00] <alice> one
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-04.log
new file mode 100644
index 0000000..82979bc
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-04.log
@@ -0,0 +1 @@
+[03:30:41] *** Joins: tester (~tester@localhost)
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-05.log
new file mode 100644
index 0000000..182b36b
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/#b/2024-06-05.log
@@ -0,0 +1,3 @@
+[01:00:00] *** dummy is now known as Guest1234
+[02:00:00] *** bob changes topic to 'Hear ye...'
+[03:00:00] *** ChanServ sets mode: +o bob
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-04.log
new file mode 100644
index 0000000..fb96300
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-04.log
@@ -0,0 +1,2 @@
+[10:24:42] <tester> send alice Thanks.
+[10:24:42] -MemoServ- The memo has been successfully sent to alice.
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-05.log
new file mode 100644
index 0000000..a7de83f
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/memoserv/2024-06-05.log
@@ -0,0 +1 @@
+[08:00:00] -MemoServ- alice has read your memo, which was sent at Jun 04 10:24:42 2024 +0000
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-04.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-04.log
new file mode 100644
index 0000000..1772e00
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-04.log
@@ -0,0 +1,2 @@
+[16:09:20] <tester> one
+[16:09:20] <tester> one
diff --git a/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-05.log b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-05.log
new file mode 100644
index 0000000..bff2ccb
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/expect_a2/soju-test-user/testnet/tester/2024-06-05.log
@@ -0,0 +1,6 @@
+[01:00:00] <tester> two
+[01:00:00] <tester> two
+[02:00:00] <tester> three
+[02:00:00] <tester> three
+[03:00:00] <tester> four
+[03:00:00] <tester> four
diff --git a/contrib/unmigrate-logs/testdata/raw_align.log b/contrib/unmigrate-logs/testdata/raw_align.log
new file mode 100644
index 0000000..d618a7f
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/raw_align.log
@@ -0,0 +1,3 @@
+# 2024-06-06.log
+@time=2024-06-06T01:00:00.000Z;account=bob :bob!~u@u/bob QUIT :Bye
+@time=2024-06-06T01:00:00.000Z;account=alice :alice!~u@u/alice QUIT :Ping timeout: 245 seconds
diff --git a/contrib/unmigrate-logs/testdata/raw_chan_a.log b/contrib/unmigrate-logs/testdata/raw_chan_a.log
new file mode 100644
index 0000000..28ae543
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/raw_chan_a.log
@@ -0,0 +1,15 @@
+# 2024-06-04.log
+@time=2024-06-04T03:30:41.891Z;account=tester :tester!~tester@localhost JOIN #a tester :tester
+@time=2024-06-04T06:37:14.603Z;account=bob :bob!~u@u/bob JOIN #a bob unknown
+@time=2024-06-04T06:45:59.473Z;account=alice :alice!~u@u/alice JOIN #a alice :Alice
+@time=2024-06-04T16:04:37.191Z;account=bob :bob!~u@u/bob QUIT :Remote host closed the connection
+@time=2024-06-04T16:13:29.735Z;account=bob :bob!~u@u/bob JOIN #a bob unknown
+
+# 2024-06-05.log
+@time=2024-06-05T01:00:00.000Z;account=bob :bob!~u@u/bob QUIT :Ping timeout: 246 seconds
+
+# %1
+@time=2024-06-05T02:00:00.000Z;account=bob :bob!~u@u/bob JOIN #a bob unknown
+
+# %2
+@time=2024-06-05T03:00:00.000Z;account=alice :alice!~u@u/alice PRIVMSG #a :one
diff --git a/contrib/unmigrate-logs/testdata/raw_chan_b.log b/contrib/unmigrate-logs/testdata/raw_chan_b.log
new file mode 100644
index 0000000..aa88767
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/raw_chan_b.log
@@ -0,0 +1,11 @@
+# 2024-06-04.log
+@time=2024-06-04T03:30:41.891Z;account=tester :tester!~tester@localhost JOIN #b tester :tester
+
+# 2024-06-05.log
+@time=2024-06-05T01:00:00.000Z :dummy!~u@u/dummy NICK Guest1234
+
+# %1
+@time=2024-06-05T02:00:00.000Z;account=bob :bob!~u@u/bob TOPIC #b :Hear ye...
+
+# %2
+@time=2024-06-05T03:00:00.000Z :ChanServ!ChanServ@services MODE #b +o bob
diff --git a/contrib/unmigrate-logs/testdata/raw_query_a.log b/contrib/unmigrate-logs/testdata/raw_query_a.log
new file mode 100644
index 0000000..deedfa1
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/raw_query_a.log
@@ -0,0 +1,8 @@
+# 2024-06-04.log
+@time=2024-06-04T10:24:42.126Z;account=tester :tester!~tester@localhost PRIVMSG MemoServ :send alice Thanks.
+@time=2024-06-04T10:24:42.356Z :MemoServ!MemoServ@services NOTICE tester :The memo has been successfully sent to alice.
+
+# 2024-06-05.log
+
+# %1
+@time=2024-06-05T08:00:00.000Z :MemoServ!MemoServ@services NOTICE tester :alice has read your memo, which was sent at Jun 04 10:24:42 2024 +0000
diff --git a/contrib/unmigrate-logs/testdata/raw_query_b.log b/contrib/unmigrate-logs/testdata/raw_query_b.log
new file mode 100644
index 0000000..9d6aa73
--- /dev/null
+++ b/contrib/unmigrate-logs/testdata/raw_query_b.log
@@ -0,0 +1,16 @@
+# 2024-06-04.log
+@time=2024-06-04T16:09:20.071Z;account=tester :tester!~tester@localhost PRIVMSG tester one
+@time=2024-06-04T16:09:20.107Z;account=tester :tester!~tester@localhost PRIVMSG tester one
+
+# 2024-06-05.log
+@time=2024-06-05T01:00:00.001Z;account=tester :tester!~tester@localhost PRIVMSG tester :two
+@time=2024-06-05T01:00:00.002Z;account=tester :tester!~tester@localhost PRIVMSG tester :two
+
+# %1
+@time=2024-06-05T02:00:00.001Z;account=tester :tester!~tester@localhost PRIVMSG tester three
+@time=2024-06-05T02:00:00.002Z;account=tester :tester!~tester@localhost PRIVMSG tester three
+
+# %2
+@time=2024-06-05T03:00:00.001Z;account=tester :tester!~tester@localhost PRIVMSG tester :four
+@time=2024-06-05T03:00:00.002Z;account=tester :tester!~tester@localhost PRIVMSG tester :four
+
--
2.45.2