This implements a new format to store SMS/MMS/VVM.

I keep a flat file for each conversation, named by the
cannonical name of the contact (Phone number or sorted phone
numbers for a group). The message bodies are indented by a
single space. Headers are non-indented lines of the form:

```
<date><tab>"sent"|"recv"<tab>"SMS"|"MMS"|"VVM"<tab><unique_id><tab><sender><tab><recipient>[<tab><tab-separated additional recipients list>]
```

I've added the possibility to add comments to the file,
using lines starting with `#`. I already use this to signal
the text log format version, but it can be very handy to add
additional info in the file (I won't develop, but it may
become useful to implement left-right conversation display
in group mms conversations).

The migration script is in the next commit.

---
 scripts/modem/sxmo_mms.sh   | 40 +++++++++++++++-----------
 scripts/modem/sxmo_modem.sh | 56 ++++++++++++++++++++++++++++++++-----
 scripts/modem/sxmo_vvm.sh   | 12 +++++---
 3 files changed, 81 insertions(+), 27 deletions(-)

diff --git a/scripts/modem/sxmo_mms.sh b/scripts/modem/sxmo_mms.sh
index 7126722..5ef375f 100644
--- a/scripts/modem/sxmo_mms.sh
+++ b/scripts/modem/sxmo_mms.sh
@@ -52,7 +52,7 @@ checkforlostmms() {
 			case "$MESSAGE_STATUS" in
 				sent)
 					stderr "This mms is status:sent."
-					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "Sent"
+					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "sent"
 					;;
 				draft)
 					stderr "This mms is status:draft."
@@ -60,7 +60,7 @@ checkforlostmms() {
 					;;
 				received)
 					stderr "This mms is status:received."
-					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "Received"
+					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "recv"
 
 					;;
 				*)
@@ -145,7 +145,7 @@ extractmmsattachement() {
 		if [ "$ACTYPE" != "text/plain" ]; then
 			printf "$icon_att %s\n" \
 				"$(basename "$SXMO_LOGDIR/$LOGDIRNUM/attachments/$OUTFILE")" \
-				>> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+				>> " $SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 
 			printf "%s\0" "$SXMO_LOGDIR/$LOGDIRNUM/attachments/$OUTFILE"
 		fi
@@ -154,7 +154,7 @@ extractmmsattachement() {
 
 processmms() {
 	MESSAGE_PATH="$1"
-	MESSAGE_TYPE="$2" # Sent or Received
+	MESSAGE_TYPE="$2" # sent or recv
 	MESSAGE="$(mmsctl -M -o "$MESSAGE_PATH")"
 	stderr "Processing mms ($MESSAGE_PATH)."
 
@@ -181,32 +181,39 @@ processmms() {
 		fi
 	fi
 
-	if [ "$MESSAGE_TYPE" = "Sent" ]; then
+	if [ "$MESSAGE_TYPE" = "sent" ]; then
 		FROM_NUM="$MYNUM"
-	elif [ "$MESSAGE_TYPE" = "Received" ]; then
+	elif [ "$MESSAGE_TYPE" = "recv" ]; then
 		FROM_NUM="$(printf %s "$MESSAGE" | jq -r '.attrs.Sender')"
 	fi
 
-	FROM_NAME="$(sxmo_contacts.sh --name "$FROM_NUM")"
 	TO_NUMS="$(printf %s "$MESSAGE" | jq -r '.attrs.Recipients | join("\n")')"
 	# generate string of contact names, e.g., "BOB, SUZIE, SAM"
-	TO_NAMES="$(printf %s "$TO_NUMS" | xargs -n1 sxmo_contacts.sh --name | tr '\n' '\0' | xargs -0 printf "%s, " | sed 's/, $//')"
+	# TO_NAMES="$(printf %s "$TO_NUMS" | xargs -n1 sxmo_contacts.sh --name | tr '\n' '\0' | xargs -0 printf "%s, " | sed 's/, $//')"
 
 	count="$(printf "%s" "$TO_NUMS" | wc -l)"
 	if [ "$count" -gt 0 ]; then
 		# a group chat.  LOGDIRNUM = all numbers except one's own, sorted numerically
 		LOGDIRNUM="$(printf "%b\n%s\n" "$TO_NUMS" "$FROM_NUM" | grep -v "^$MYNUM$" | sort -u | grep . | xargs printf %s)"
 		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
-		printf "%s Group MMS from %s to %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$TO_NAMES" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
+			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+		fi
+		sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$(printf %s "$TO_NUMS" | tr '\n' '\t')" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+		# printf "%s Group MMS from %s to %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$TO_NAMES" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 	else
 		# not a group chat
-		if [ "$MESSAGE_TYPE" = "Sent" ]; then
+		if [ "$MESSAGE_TYPE" = "sent" ]; then
 			LOGDIRNUM="$TO_NUMS"
-		elif [ "$MESSAGE_TYPE" = "Received" ]; then
+		elif [ "$MESSAGE_TYPE" = "recv" ]; then
 			LOGDIRNUM="$FROM_NUM"
 		fi
 		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
-		printf "%s MMS from %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
+			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+		fi
+		printf "%s\n" "$(sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$TO_NUMS")" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+		# printf "%s MMS from %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 	fi
 
 	stderr "$MESSAGE_TYPE MMS ($MMS_FILE) from number $LOGDIRNUM"
@@ -219,9 +226,9 @@ processmms() {
 
 	mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM/attachments"
 
-	if [ "$MESSAGE_TYPE" = "Received" ]; then
+	if [ "$MESSAGE_TYPE" = "recv" ]; then
 		printf "%s\trecv_mms\t%s\t%s\n" "$DATE" "$LOGDIRNUM" "$MMS_FILE" >> "$SXMO_LOGDIR/modemlog.tsv"
-	elif [ "$MESSAGE_TYPE" = "Sent" ]; then
+	elif [ "$MESSAGE_TYPE" = "sent" ]; then
 		printf "%s\tsent_mms\t%s\t%s\n" "$DATE" "$LOGDIRNUM" "$MMS_FILE" >> "$SXMO_LOGDIR/modemlog.tsv"
 	else
 		stderr "Unknown message type: $MESSAGE_TYPE for $MMS_FILE"
@@ -236,9 +243,10 @@ processmms() {
 		TEXT="<Empty>"
 	fi
 
-	printf "%b\n\n" "$TEXT" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+	printf %b "$TEXT\n" | sxmo_modem.sh formatbody >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 
-	if [ "$MESSAGE_TYPE" = "Received" ]; then
+	if [ "$MESSAGE_TYPE" = "recv" ]; then
+		FROM_NAME="$(sxmo_contacts.sh --name "$FROM_NUM")"
 		[ "$FROM_NAME" = "???" ] && FROM_NAME="$FROM_NUM"
 		if [ -z "$SXMO_DISABLE_SMS_NOTIFS" ]; then
 			[ -n "$OPEN_ATTACHMENTS_CMD" ] && TEXT="$icon_att $TEXT"
diff --git a/scripts/modem/sxmo_modem.sh b/scripts/modem/sxmo_modem.sh
index b268548..218ba1d 100644
--- a/scripts/modem/sxmo_modem.sh
+++ b/scripts/modem/sxmo_modem.sh
@@ -33,6 +33,40 @@ lookupnumberfromcallid() {
 		tr -d ' '
 }
 
+_gen_id(){
+	cat /dev/random | tr -dc "[:alnum:]" | head -c 15
+}
+
+formatheader(){
+	TIME="$1"
+	shift
+	DIRECTION="$1"
+	shift
+	TYPE="$1"
+	shift
+	SENDER="$1"
+	shift
+	RECIPIENTS=""
+	while [ "$#" != "0" ]; do
+		RECIPIENTS="$RECIPIENTS	$1"
+		shift
+	done
+	RECIPIENTS="$(echo "$RECIPIENTS" | sed 's/^\t//')"
+	printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$TIME" "$DIRECTION" "$TYPE" "$(_gen_id)" "$SENDER" "$RECIPIENTS"
+}
+
+formatbody(){
+	while IFS= read -r line; do
+		printf " %b\n" "$line"
+	done
+	printf "\n"
+}
+
+formattext(){
+	formatheader "$@"
+	formatbody
+}
+
 checkforfinishedcalls() {
 	#find all finished calls
 	for FINISHEDCALLID in $(
@@ -140,11 +174,11 @@ checkforincomingcalls() {
 }
 
 checkfornewtexts() {
-	TEXTIDS="$(
-		mmcli -m any --messaging-list-sms |
-		grep -Eo '/SMS/[0-9]+ \(received\)' |
-		grep -Eo '[0-9]+'
-	)"
+TEXTIDS="$(
+mmcli -m any --messaging-list-sms |
+grep -Eo '/SMS/[0-9]+ \(received\)' |
+grep -Eo '[0-9]+'
+)"
 	echo "$TEXTIDS" | grep -v . && return
 
 	# Loop each textid received and read out the data into appropriate logfile
@@ -205,8 +239,12 @@ checkfornewtexts() {
 		# the sms wap number?
 		if cut -f1 "$SXMO_BLOCKFILE" 2>/dev/null | grep -q "^$NUM$"; then
 			mkdir -p "$SXMO_BLOCKDIR/$NUM"
+			if [ ! -e "$SXMO_BLOCKDIR/$NUM/sms.txt" ]; then
+				echo "# sms_log_version: 1" > "$SXMO_BLOCKDIR/$NUM/sms.txt"
+			fi
 			stderr "BLOCKED text from number: $NUM (TEXTID: $TEXTID)"
-			printf %b "Received from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
+			printf %b "$(echo "$TEXT" | formattext "$TIME" Recv SMS "" "$NUM")" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
+			# printf %b "Received from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
 			printf %b "$TIME\trecv_txt\t$NUM\t${#TEXT} chars\n" >> "$SXMO_BLOCKDIR/modemlog.tsv"
 			mmcli -m any --messaging-delete-sms="$TEXTID"
 			continue
@@ -234,9 +272,13 @@ checkfornewtexts() {
 		fi
 
 		mkdir -p "$SXMO_LOGDIR/$NUM"
+		if [ ! -e "$SXMO_LOGDIR/$NUM/sms.txt" ]; then
+			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$NUM/sms.txt"
+		fi
 		stderr "Text from number: $NUM (TEXTID: $TEXTID)"
-		printf %b "Received SMS from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_LOGDIR/$NUM/sms.txt"
 		printf %b "$TIME\trecv_txt\t$NUM\t${#TEXT} chars\n" >> "$SXMO_LOGDIR/modemlog.tsv"
+		printf %b "$(echo "$TEXT" | formattext "$TIME" Recv SMS "" "$NUM")" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
+		# printf %b "Received SMS from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_LOGDIR/$NUM/sms.txt"
 		mmcli -m any --messaging-delete-sms="$TEXTID"
 		CONTACTNAME=$(sxmo_contacts.sh --name "$NUM")
 		[ "$CONTACTNAME" = "???" ] && CONTACTNAME="$NUM"
diff --git a/scripts/modem/sxmo_vvm.sh b/scripts/modem/sxmo_vvm.sh
index 7b82f49..4d8c636 100644
--- a/scripts/modem/sxmo_vvm.sh
+++ b/scripts/modem/sxmo_vvm.sh
@@ -17,9 +17,6 @@ processvvm() {
 	VVM_ID="$3" # unique id assigned to voice mail from vvmd
 	VVM_ATTACHMENT="$4" # full path + filename of amr file
 	VVM_FILE="$SXMO_LOGDIR/$VVM_SENDER/attachments/$(basename "$VVM_ATTACHMENT")"
-	VVM_SENDER_NAME="$(sxmo_contacts.sh --name "$VVM_SENDER")"
-	[ "$VVM_SENDER_NAME" = "???" ] && VVM_SENDER_NAME="$VVM_SENDER"
-
 	mkdir -p "$SXMO_LOGDIR/$VVM_SENDER/attachments"
 
 	printf "%s\trecv_vvm\t%s\t%s\n" "$VVM_DATE" "$VVM_SENDER" "$VVM_ID" >> "$SXMO_LOGDIR/modemlog.tsv"
@@ -31,9 +28,16 @@ processvvm() {
 		exit 1
 	fi
 
-	printf "Received Voice Mail from %s at %s:\n%s %s\n\n" "$VVM_SENDER_NAME" "$VVM_DATE" "$icon_att" "$(basename "$VVM_FILE")" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
+	if [ ! -e "$SXMO_LOGDIR/$VVM_SENDER/sms.txt" ]; then
+		echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
+	fi
+	sxmo_modem.sh formatheader "$VVM_DATE" recv "VVM" "" "$VVM_SENDER" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
+	printf "%s %s\n\n" "$icon_att" "$(basename "$VVM_FILE")" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
 
 	if [ -z "$SXMO_DISABLE_SMS_NOTIFS" ]; then
+		VVM_SENDER_NAME="$(sxmo_contacts.sh --name "$VVM_SENDER")"
+		[ "$VVM_SENDER_NAME" = "???" ] && VVM_SENDER_NAME="$VVM_SENDER"
+
 		sxmo_notificationwrite.sh \
 			random \
 			"sxmo_open.sh '$VVM_FILE'" \
-- 
2.35.1

Migration script and tailtextlog hook for the new sms format.

This is WIP...

Notes:

1. The migration script allows rollbacks, but it's not very useful as
   sxmo_modem* will use the new format anyway. They should be changed
   to detect the format of the file and save texts accordingly.

2. The tailtextlog hook is not as pretty as my previous experiments. They
   are broken when there is wide characters in the message (fail to
   right-justify). 

---
 .build.yml                                    |   2 +-
 .../default_hooks/sxmo_hook_tailtextlog.sh    | 308 +++++++++++++++++-
 migrations/1.9.0.1.sh                         | 207 ++++++++++++
 scripts/modem/sxmo_mms.sh                     |   4 +-
 scripts/modem/sxmo_modem.sh                   |   6 +-
 scripts/modem/sxmo_vvm.sh                     |   2 +-
 6 files changed, 518 insertions(+), 11 deletions(-)
 create mode 100644 migrations/1.9.0.1.sh

diff --git a/.build.yml b/.build.yml
index 10245f3..d206f30 100644
--- a/.build.yml
+++ b/.build.yml
@@ -8,7 +8,7 @@ sources:
 tasks:
   - shellcheck: |
       cd sxmo-utils
-      shellcheck -x scripts/*/*.sh
+      shellcheck -x scripts/*/*.sh -x migrations/*.sh
   - shellspec: |
       cd sxmo-utils
       shellspec
diff --git a/configs/default_hooks/sxmo_hook_tailtextlog.sh b/configs/default_hooks/sxmo_hook_tailtextlog.sh
index 9a2b760..27ebcca 100755
--- a/configs/default_hooks/sxmo_hook_tailtextlog.sh
+++ b/configs/default_hooks/sxmo_hook_tailtextlog.sh
@@ -3,10 +3,310 @@
 # This hook displays the sms log for a numbers passed as $1
 
 . sxmo_common.sh
-NUMBER="$1"
-CONTACTNAME="$(sxmo_contacts.sh | grep ": ${NUMBER}$" | cut -d: -f1)"
-[ "???" = "$CONTACTNAME" ] && CONTACTNAME="$CONTACTNAME ($NUMBER)"
 
-TERMNAME="$NUMBER SMS" sxmo_terminal.sh sh -c "tail -n9999 -f \"$SXMO_LOGDIR/$NUMBER/sms.txt\" | sed \"s|$NUMBER|$CONTACTNAME|g\""
+SXMO_TEXT_BACKLOG="${SXMO_TEXT_BACKLOG:-500}"
 
+num2contact(){
+	contact="$(sxmo_contacts.sh --all | grep -e ": $@$" | cut -d ":" -f 1)"
+	if [ -z "$contact" ]; then
+		contact="$@"
+	fi
+	return contact
+}
 
+formatlog(){
+	NUMBER="$1"
+	SMSLOG="$SXMO_LOGDIR/$NUMBER/sms.txt"
+	if [ ! -e "$SMSLOG" ]; then
+		mkdir -p "$(dirname "$SMSLOG")"
+		echo "# text_log_version: 1" > "$SMSLOG"
+	fi
+	HEADER_PLACEHOLDER="$(cat /dev/random | head -c 50 | base64)"
+
+	# first, we remove headers (replaced by HEADER_PLACEHOLDER). We `tac`
+	# the file to count the number of messages we display
+	msgs="$(mktemp)"
+	headers="$(mktemp)"
+	trap "rm -f $msgs" INT TERM EXIT
+	trap "rm -f $headers" INT TERM EXIT
+	tac "$SMSLOG" | awk \
+		-v headers="$headers" \
+		-v BACKLOG="$SXMO_TEXT_BACKLOG" \
+		-v HEADER_PLACEHOLDER="$HEADER_PLACEHOLDER" \
+		'
+		# message content
+		/^ .*/ {
+			print $0
+			next
+		}
+		# comment... ignore
+		/^#.*/ {
+			next
+		}
+		# header
+		{
+			print hd > headers
+			print HEADER_PLACEHOLDER
+			BACKLOG--
+			if(BACKLOG == 0) exit
+		}
+
+	' | tac > "$msgs"
+
+	# sort the messages based on date (not TZ aware, yet), format headers and
+	# print content.
+	tac "$headers" | \
+	awk \
+		-v msgs_file="$msgs" \
+		-v NUM2CONTACT="$SCRIPT num2contact " \
+		-v HEADER_PLACEHOLDER="$HEADER_PLACEHOLDER" \
+		'
+		# quicksort
+		function less_than(left, right) {
+			return "" left <= "" right
+		}
+		function quicksort(data, left, right, i, last){
+			if (left >= right)
+				return
+			quicksort_swap(data, left, int((left + right) / 2))
+			last = left
+			for (i = left + 1; i <= right; i++)
+				if (less_than(data[i], data[left]))
+					quicksort_swap(data, ++last, i)
+			quicksort_swap(data, left, last)
+			quicksort(data, left, last - 1)
+			quicksort(data, last + 1, right)
+		}
+		function quicksort_swap(data, i, j, temp){
+			temp = data[i]
+			data[i] = data[j]
+			data[j] = temp
+		}
+
+		# memoized contact name resolution
+		function num2contact(num,  contact){
+			if(! contacts[num]){
+				NUM2CONTACT num | getline contact
+				contacts[num] = contact
+				close(NUM2CONTACT num)
+			}
+			return contacts[num]
+		}
+		function format_header(hd,  parts, date, dir, msg_type, from, postfix){
+			split(hd, parts, "\t")
+			date = parts[1]
+			from = ""
+			if (parts[2] == "sent") {
+				dir = "<<<< Sent "
+				postfix = " <<<<"
+			}
+			else {
+				dir = ">>>> Received "
+				postfix = " >>>>"
+				if(parts[7]) {
+					# multiple recipients -> group text -> print sender s name
+					from = " from " num2contact parts[5]
+				}
+			}
+			msg_type = dir[3]
+			printf("%s%s%s%s", dir, msg_type, from, postfix)
+		}
+		
+		BEGIN {
+			split("", headers)
+			split("", header_keys)
+			split("", messages)
+			headerindex = 0
+			msgindex = 0
+			# contact names cache
+			split("", contacts)
+		}
+		FILENAME == "-" {
+			headerindex++
+			headers[headerindex] = $0
+			header_keys[$0] = headerindex
+			next
+		}
+		$0 == HEADER_PLACEHOLDER {
+			msgindex++
+			next
+		}
+		{
+			if (!messages[msgindex]) messages[msgindex] = $0
+			else messages[msgindex] = messages[msgindex] "\n" $0
+		}
+		END {
+			for (key in header_keys) { keys[n++]=key }
+			quicksort(keys, 0, n-1)
+			for (i = 0; i < n; i++) {
+				idx = header_keys[keys[i]]
+				print format_header(headers[idx])
+				print messages[idx]
+			}
+		}
+		' - "$msgs" #|\
+	exit
+
+	 # format headers and align messages depending on the sender
+	 # (right = me, left = other)
+	 awk \
+		-v WIDTH="$WIDTH" \
+		-v TODAY="$TODAY" \
+		-v YESTERDAY="$YESTERDAY" \
+		-v TWO_DAYS="$TWO_DAYS" \
+		-v MAX_WIDTH="$MAX_WIDTH" \
+		-v CONTACTNAME="$CONTACTNAME" \
+		-v MSG_COUNT="$(wc -l "$headers")" '
+		# utils
+		function repeat(c, num) {
+			s = ""
+			for(i=0;i<num;i++) s = s c;
+			return s
+		}
+
+		function min(a, b){
+			if(a<b) return a
+			return b
+		}
+
+		function max(a, b){
+			if(a>b) return a
+			return b
+		}
+
+		function print_msg(){
+			if(them){
+				return
+			} else {
+				max_length = 0
+				for(ii=0;ii<length(msg);ii++){
+					max_length = max(max_length, length(msg[ii]))
+				}
+				max_length = min(max_length, MAX_WIDTH)
+				padding = repeat(" ", WIDTH - max_length)
+				for(ii=0;ii<length(msg);ii++){
+					# print "@" msg[ii] "@"
+					print padding msg[ii]
+				}
+				split("", msg)
+			}
+		}
+
+		# format the date and add human day names ("today", "yesterday", "2 days ago")
+		function get_human_date(datetime){
+			date = gensub("^.*( [0-9-]+)T.*", "\\1", "g", datetime)
+			if(date ~ "^ " TODAY "$") {
+				return ""
+			}
+			if(date ~ "^ " YESTERDAY "$") {
+				return " yesterday at"
+			}
+			if(date ~ "^ " TWO_DAYS "$") {
+				return " 2 days ago at"
+			}
+			return date
+		}
+
+		# Parse the message header to extract the time of the message.
+		# Remove the timzone info
+		function get_time(datetime){
+			return gensub("^.* [0-9-]+T([0-9:]+).*", "\\1", "g", datetime)
+		}
+
+		BEGIN {
+			them = 1
+			split("", msg)
+			msg_count=int(MSG_COUNT)
+			timezone_re="[+-][0-9]{2}:?[0-9]{2}"
+		}
+
+		# Parse, reformat and print message headers
+		/^[0-9]+\tSent SMS|MMS to .* at [0-9T:+-]+:$/{
+			# dump the preceding message if needed
+			print_msg()
+			them=0
+			title = " [" MSG_COUNT-- "]" get_human_date($0) " " get_time($0) " --"
+			prefix_len=WIDTH - length(title)
+			print "\033[37m" repeat("-", prefix_len) title "\033[m"
+			next
+		}
+
+		/^[0-9]+\tReceived SMS|MMS from .* at [0-9T:+-]+:$/{
+			print_msg()
+			them=1
+			title = "-- [" MSG_COUNT-- "]" get_human_date($0) " " get_time($0) " "
+			postfix_len=WIDTH - length(title)
+			print "\033[37m" title repeat("-", postfix_len) "\033[m"
+			next
+		}
+
+		# Not a header
+		{
+			if(them){
+				# Print early, it is left aligned
+				print
+				next
+			} else {
+				# Append the line to the message array
+				# to pad when we konw the longuest line.
+				msg[length(msg)] = $0
+			}
+		}
+
+		END {
+			# dump the last message, as no header triggerd the dump.
+			print_msg()
+			# add a title (the contact name)
+			title = "--[ " CONTACTNAME " ]"
+			postfix_len=WIDTH - length(title)
+			title = "\033[34m" title repeat("-", postfix_len) "\033[m"
+			printf("%s", title)
+		}
+	'
+}
+
+textlogloop(){
+	# The loop watches the modem log file (comparing md5 sums, we could
+	# use inotify for this, too).
+	# On change, format the messages in a temp file, clear the term and
+	# dump the tempfile.
+
+	NUMBER="$1"
+	SMSLOG="$SXMO_LOGDIR/$NUMBER/sms.txt"
+
+	resultfile="$(mktemp)"
+	trap "rm -f '$resultfile'" INT TERM EXIT
+
+	dumplogs() {
+		SUM="1234 $SMSLOG"
+		while ! echo "$SUM" | md5sum -c - >/dev/null 2>&1; do
+			tput civis
+			TERM_LINES="$(tput lines)"
+			formatlog "$NUMBER" > "$resultfile"
+			printf "\033c"
+			cat "$resultfile"
+			SUM="$(md5sum "$SMSLOG")"
+		done
+	}
+
+	dumplogs
+
+	while inotifywait -e modify "$SMSLOG" > /dev/null 2>&1; do
+		dumplogs
+	done
+}
+
+if [ "1" == "$#" ]; then
+	NUMBER="$1"
+	if grep -qe "^# text_log_version: 1$" "$SXMO_LOGDIR/$NUMBER/sms.txt"; then
+		# migrated to versioned (v1) format
+		TERMNAME="$NUMBER SMS" sxmo_terminal.sh sh -c "$0 textlogloop '$NUMBER'"
+	else
+		# old unversioned format. Should we send a notification?
+		CONTACTNAME="$(sxmo_contacts.sh | grep ": ${NUMBER}$" | cut -d: -f1)"
+		[ "???" = "$CONTACTNAME" ] && CONTACTNAME="$CONTACTNAME ($NUMBER)"
+		TERMNAME="$NUMBER SMS" sxmo_terminal.sh sh -c "tail -n9999 -f \"$SXMO_LOGDIR/$NUMBER/sms.txt\" | sed \"s|$NUMBER|$CONTACTNAME|g\""
+	fi
+else
+	"$@"
+fi
diff --git a/migrations/1.9.0.1.sh b/migrations/1.9.0.1.sh
new file mode 100644
index 0000000..c2f1957
--- /dev/null
+++ b/migrations/1.9.0.1.sh
@@ -0,0 +1,207 @@
+#!/bin/sh
+
+# Migrate text logs (~/.local/share/sxmo/{modem,block}/*/sms.txt) to
+# the new storage version. This script creates a backup file for
+# each migrated file.
+
+# shellcheck source=scripts/core/sxmo_common.sh
+. "$(which sxmo_common.sh)"
+
+SCRIPT="$0"
+
+contact2num (){
+	contact="$1"
+	if [ "$contact" = "???" ]; then
+		printf "???"
+	else
+		num="$(sxmo_contacts.sh --all | grep -e "^$contact:" | head -1 | cut -d":" -f2- | sed 's/^\s*//')"
+		if [ -z "$num" ]; then
+			printf "???"
+		else
+			printf "%s" "$num"
+		fi
+	fi
+}
+
+gen_id(){
+	base64 < /dev/random | tr -dc "[:alnum:]" | head -c 1500
+}
+
+get_files(){
+	find "$SXMO_BLOCKDIR" "$SXMO_LOGDIR" -name sms.txt |\
+		while read -r f; do
+			if ! grep -qe "^# text_log_version: 1$" "$f"; then
+				echo "$f"
+			fi
+		done
+}
+
+get_bkp_files(){
+	find "$SXMO_BLOCKDIR" "$SXMO_LOGDIR" -name sms.txt.migrate.bkp
+}
+
+migratefile() {
+	# set -x
+	file="$1"
+	CONTACTDIR="$(basename "$(dirname "$file")")"
+	output="$file.migrate.tmp"
+	bkp="$file.migrate.bkp"
+	CONTACTS="$(echo "$CONTACTDIR" | sed 's/\+/\t+/g' | sed 's/^\t//')"
+
+	# this logic is the same as the logic found in sxmo_mms.sh
+	MYNUM="$(sxmo_contacts.sh --me)"
+	if [ -z "$MYNUM" ]; then
+		sxmo_log "The migration script can't infer your phone number."
+		sxmo_log "This probably means you need to configure the Me contact."
+		sxmo_log "We will use a fake number (+12345670000) instead."
+		MYNUM="+12345670000"
+	fi
+
+	cp "$file" "$bkp"
+	sxmo_log "backup file $bkp created"
+
+	busybox awk \
+	-v CONTACT2NUM="$SCRIPT contact2num " \
+	-v CONTACTS="$CONTACTS" \
+	-v MYNUM="$MYNUM" \
+	-v GENID="$SCRIPT gen_id" \
+	'
+	BEGIN{
+		# phone number cache
+		split("", nums)
+
+		# cache of MMS Group recipient list without the sender
+		split("", num_lists)
+
+		# Calling an external procces is very costly. We generate
+		# a 1500 chars long random id at once and splice it.
+		# (see get_id() definition)
+		id_pool = ""
+
+		# add a format version number.
+		print "# text_log_version: 1"
+	}
+
+	function get_id(nxt){
+		if (id_pool == "") {
+			GENID | getline id_pool
+			close(GENID)
+		}
+		nxt = substr(id_pool, 0, 15)
+		id_pool = substr(id_pool, 16)
+		return nxt
+	}
+
+	function contact2num(contact){
+		if(! nums[contact]){
+			CONTACT2NUM contact | getline num
+			nums[contact] = num
+			close(CONTACT2NUM contact)
+		}
+		return nums[contact]
+	}
+
+	function rm_from_list(num, lst,  key, newlst, lstarr, i){
+		key = num " " lst
+		if(! num_lists[key]){
+			split(lst, lstarr, "\t")
+			newlst = ""
+			for(i in lstarr) {
+				if(num != lstarr[i]){
+					if(newlst == ""){
+						newlst = lstarr[i]
+					} else {
+						newlst = newlst "\t" lstarr[i]
+					}
+				}
+			}
+			num_lists[key] = newlst
+		}
+		return num_lists[key]
+	}
+
+	function get_date(hd) {
+		return gensub("^.* at (.*):$", "\\1", "g", hd)
+	}
+
+	# SMS
+	/^Sent SMS to .* at [0-9T:+-]+:$/ {
+		sender = MYNUM
+		recipient = CONTACTS
+		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "sent", "SMS", get_id(), sender, recipient)
+		next
+	}
+	/^Received SMS from .* at [0-9T:+-]+:$/ {
+		sender = CONTACTS
+		recipient = MYNUM
+		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "SMS", get_id(), sender, recipient)
+		next
+	}
+
+	# 1-to-1 MMS
+	/^Sent MMS from .* at [0-9T:+-]+:$/ {
+		sender = MYNUM
+		recipient = CONTACTS
+		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "sent", "MMS", get_id(), sender, recipient)
+		next
+	}
+	/^Received MMS from .* at [0-9T:+-]+:$/ {
+		sender = CONTACTS
+		recipient = MYNUM
+		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "MMS", get_id(), sender, recipient)
+		next
+	}
+
+	# Voice Mail
+	/^Received Voice Mail from .* at [0-9T:+-]+:$/ {
+		sender = CONTACTS
+		recipient = MYNUM
+		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "VVM", get_id(), sender, recipient)
+		next
+	}
+
+	# Group MMS
+	/^Sent Group MMS from .+ to .+ at [0-9T:+-]+:$/ {
+		sender = MYNUM
+		recipient = CONTACTS
+		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "sent", "MMS", get_id(), sender, recipient)
+		next
+	}
+	/^Received Group MMS from .* to .* at [0-9T:+-]+:$/ {
+		sender = gensub("Received Group MMS from (.+) to .+ at [0-9T:+-]+:$", "\\1", "g", $0)
+		sender = contact2num(sender)
+		recipient = rm_from_list(sender, CONTACTS "\t" MYNUM)
+		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "MMS", get_id(), sender, recipient)
+		next
+	}
+
+	# message content
+	{
+		print " " $0
+	}
+	' "$file" > "$output"
+	mv "$output" "$file"
+}
+
+migrate_all(){
+	sxmo_log "Migrate all sms log files."
+	sxmo_log "A backup is created. In case of unexpected behaviour, revert with:"
+	sxmo_log "sh $SCRIPT rollback"
+	get_files | while read -r file; do
+		sxmo_log "Migarting $file"
+		migratefile "$file"
+	done
+}
+
+rollback(){
+	get_bkp_files | while read -r src; do
+		target="${src%.migrate.bkp}"
+		mv "$src" "$target"
+	done
+}
+
+if [ "0" = "$#" ]; then
+	migrate_all
+else
+	"$@"
+fi
diff --git a/scripts/modem/sxmo_mms.sh b/scripts/modem/sxmo_mms.sh
index 5ef375f..0a54a65 100644
--- a/scripts/modem/sxmo_mms.sh
+++ b/scripts/modem/sxmo_mms.sh
@@ -197,7 +197,7 @@ processmms() {
 		LOGDIRNUM="$(printf "%b\n%s\n" "$TO_NUMS" "$FROM_NUM" | grep -v "^$MYNUM$" | sort -u | grep . | xargs printf %s)"
 		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
 		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
-			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+			echo "# text_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 		fi
 		sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$(printf %s "$TO_NUMS" | tr '\n' '\t')" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 		# printf "%s Group MMS from %s to %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$TO_NAMES" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
@@ -210,7 +210,7 @@ processmms() {
 		fi
 		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
 		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
-			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
+			echo "# text_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 		fi
 		printf "%s\n" "$(sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$TO_NUMS")" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
 		# printf "%s MMS from %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
diff --git a/scripts/modem/sxmo_modem.sh b/scripts/modem/sxmo_modem.sh
index 218ba1d..ac4fd91 100644
--- a/scripts/modem/sxmo_modem.sh
+++ b/scripts/modem/sxmo_modem.sh
@@ -34,7 +34,7 @@ lookupnumberfromcallid() {
 }
 
 _gen_id(){
-	cat /dev/random | tr -dc "[:alnum:]" | head -c 15
+	base64 < /dev/random | tr -dc "[:alnum:]" | head -c 1500
 }
 
 formatheader(){
@@ -240,7 +240,7 @@ grep -Eo '[0-9]+'
 		if cut -f1 "$SXMO_BLOCKFILE" 2>/dev/null | grep -q "^$NUM$"; then
 			mkdir -p "$SXMO_BLOCKDIR/$NUM"
 			if [ ! -e "$SXMO_BLOCKDIR/$NUM/sms.txt" ]; then
-				echo "# sms_log_version: 1" > "$SXMO_BLOCKDIR/$NUM/sms.txt"
+				echo "# text_log_version: 1" > "$SXMO_BLOCKDIR/$NUM/sms.txt"
 			fi
 			stderr "BLOCKED text from number: $NUM (TEXTID: $TEXTID)"
 			printf %b "$(echo "$TEXT" | formattext "$TIME" Recv SMS "" "$NUM")" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
@@ -273,7 +273,7 @@ grep -Eo '[0-9]+'
 
 		mkdir -p "$SXMO_LOGDIR/$NUM"
 		if [ ! -e "$SXMO_LOGDIR/$NUM/sms.txt" ]; then
-			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$NUM/sms.txt"
+			echo "# text_log_version: 1" > "$SXMO_LOGDIR/$NUM/sms.txt"
 		fi
 		stderr "Text from number: $NUM (TEXTID: $TEXTID)"
 		printf %b "$TIME\trecv_txt\t$NUM\t${#TEXT} chars\n" >> "$SXMO_LOGDIR/modemlog.tsv"
diff --git a/scripts/modem/sxmo_vvm.sh b/scripts/modem/sxmo_vvm.sh
index 4d8c636..00b1b30 100644
--- a/scripts/modem/sxmo_vvm.sh
+++ b/scripts/modem/sxmo_vvm.sh
@@ -29,7 +29,7 @@ processvvm() {
 	fi
 
 	if [ ! -e "$SXMO_LOGDIR/$VVM_SENDER/sms.txt" ]; then
-		echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
+		echo "# text_log_version: 1" > "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
 	fi
 	sxmo_modem.sh formatheader "$VVM_DATE" recv "VVM" "" "$VVM_SENDER" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
 	printf "%s %s\n\n" "$icon_att" "$(basename "$VVM_FILE")" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
-- 
2.35.1

sxmo-utils/patches/.build.yml: SUCCESS in 21s

[use new sms log format][0] from [Bruno Dupuis][1]

[0]: https://lists.sr.ht/~mil/sxmo-devel/patches/29935
[1]: bruno.dupuis@protonmail.com

✓ #707093 SUCCESS sxmo-utils/patches/.build.yml https://builds.sr.ht/~mil/job/707093

If you want to apply and test this, please, be very careful, it's in pre-alpha stage,
I've tested this only with dummy data on my laptop, with sxmo_* scripts stubs. It may eat your
data. At least, crate a backup of ~/.cache/share/sxmo/modem before running
sxmo_migrate.sh.