~mil/sxmo-devel

This thread contains a patchset. You're looking at the original emails, but you may wish to use the patch review UI. Review patch
3 2

[PATCH sxmo-utils 1/2] use new sms log format

Details
Message ID
<20220302131203.374082-1-bruno.dupuis@protonmail.com>
DKIM signature
missing
Download raw message
Patch: +81 -27
This implements a new format to store SMS/MMS/VVM.

I keep a flat file for each conversation, named by the
cannonical name of the contact (Phone number or sorted phone
numbers for a group). The message bodies are indented by a
single space. Headers are non-indented lines of the form:

```
<date><tab>"sent"|"recv"<tab>"SMS"|"MMS"|"VVM"<tab><unique_id><tab><sender><tab><recipient>[<tab><tab-separated additional recipients list>]
```

I've added the possibility to add comments to the file,
using lines starting with `#`. I already use this to signal
the text log format version, but it can be very handy to add
additional info in the file (I won't develop, but it may
become useful to implement left-right conversation display
in group mms conversations).

The migration script is in the next commit.

---
 scripts/modem/sxmo_mms.sh   | 40 +++++++++++++++-----------
 scripts/modem/sxmo_modem.sh | 56 ++++++++++++++++++++++++++++++++-----
 scripts/modem/sxmo_vvm.sh   | 12 +++++---
 3 files changed, 81 insertions(+), 27 deletions(-)

diff --git a/scripts/modem/sxmo_mms.sh b/scripts/modem/sxmo_mms.sh
index 7126722..5ef375f 100644
--- a/scripts/modem/sxmo_mms.sh
+++ b/scripts/modem/sxmo_mms.sh
@@ -52,7 +52,7 @@ checkforlostmms() {
			case "$MESSAGE_STATUS" in
				sent)
					stderr "This mms is status:sent."
					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "Sent"
					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "sent"
					;;
				draft)
					stderr "This mms is status:draft."
@@ -60,7 +60,7 @@ checkforlostmms() {
					;;
				received)
					stderr "This mms is status:received."
					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "Received"
					[ "$FORCE" -eq 1 ] && processmms "/org/ofono/mms/modemmanager/$line" "recv"

					;;
				*)
@@ -145,7 +145,7 @@ extractmmsattachement() {
		if [ "$ACTYPE" != "text/plain" ]; then
			printf "$icon_att %s\n" \
				"$(basename "$SXMO_LOGDIR/$LOGDIRNUM/attachments/$OUTFILE")" \
				>> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
				>> " $SXMO_LOGDIR/$LOGDIRNUM/sms.txt"

			printf "%s\0" "$SXMO_LOGDIR/$LOGDIRNUM/attachments/$OUTFILE"
		fi
@@ -154,7 +154,7 @@ extractmmsattachement() {

processmms() {
	MESSAGE_PATH="$1"
	MESSAGE_TYPE="$2" # Sent or Received
	MESSAGE_TYPE="$2" # sent or recv
	MESSAGE="$(mmsctl -M -o "$MESSAGE_PATH")"
	stderr "Processing mms ($MESSAGE_PATH)."

@@ -181,32 +181,39 @@ processmms() {
		fi
	fi

	if [ "$MESSAGE_TYPE" = "Sent" ]; then
	if [ "$MESSAGE_TYPE" = "sent" ]; then
		FROM_NUM="$MYNUM"
	elif [ "$MESSAGE_TYPE" = "Received" ]; then
	elif [ "$MESSAGE_TYPE" = "recv" ]; then
		FROM_NUM="$(printf %s "$MESSAGE" | jq -r '.attrs.Sender')"
	fi

	FROM_NAME="$(sxmo_contacts.sh --name "$FROM_NUM")"
	TO_NUMS="$(printf %s "$MESSAGE" | jq -r '.attrs.Recipients | join("\n")')"
	# generate string of contact names, e.g., "BOB, SUZIE, SAM"
	TO_NAMES="$(printf %s "$TO_NUMS" | xargs -n1 sxmo_contacts.sh --name | tr '\n' '\0' | xargs -0 printf "%s, " | sed 's/, $//')"
	# TO_NAMES="$(printf %s "$TO_NUMS" | xargs -n1 sxmo_contacts.sh --name | tr '\n' '\0' | xargs -0 printf "%s, " | sed 's/, $//')"

	count="$(printf "%s" "$TO_NUMS" | wc -l)"
	if [ "$count" -gt 0 ]; then
		# a group chat.  LOGDIRNUM = all numbers except one's own, sorted numerically
		LOGDIRNUM="$(printf "%b\n%s\n" "$TO_NUMS" "$FROM_NUM" | grep -v "^$MYNUM$" | sort -u | grep . | xargs printf %s)"
		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
		printf "%s Group MMS from %s to %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$TO_NAMES" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		fi
		sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$(printf %s "$TO_NUMS" | tr '\n' '\t')" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		# printf "%s Group MMS from %s to %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$TO_NAMES" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
	else
		# not a group chat
		if [ "$MESSAGE_TYPE" = "Sent" ]; then
		if [ "$MESSAGE_TYPE" = "sent" ]; then
			LOGDIRNUM="$TO_NUMS"
		elif [ "$MESSAGE_TYPE" = "Received" ]; then
		elif [ "$MESSAGE_TYPE" = "recv" ]; then
			LOGDIRNUM="$FROM_NUM"
		fi
		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
		printf "%s MMS from %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		fi
		printf "%s\n" "$(sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$TO_NUMS")" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		# printf "%s MMS from %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
	fi

	stderr "$MESSAGE_TYPE MMS ($MMS_FILE) from number $LOGDIRNUM"
@@ -219,9 +226,9 @@ processmms() {

	mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM/attachments"

	if [ "$MESSAGE_TYPE" = "Received" ]; then
	if [ "$MESSAGE_TYPE" = "recv" ]; then
		printf "%s\trecv_mms\t%s\t%s\n" "$DATE" "$LOGDIRNUM" "$MMS_FILE" >> "$SXMO_LOGDIR/modemlog.tsv"
	elif [ "$MESSAGE_TYPE" = "Sent" ]; then
	elif [ "$MESSAGE_TYPE" = "sent" ]; then
		printf "%s\tsent_mms\t%s\t%s\n" "$DATE" "$LOGDIRNUM" "$MMS_FILE" >> "$SXMO_LOGDIR/modemlog.tsv"
	else
		stderr "Unknown message type: $MESSAGE_TYPE for $MMS_FILE"
@@ -236,9 +243,10 @@ processmms() {
		TEXT="<Empty>"
	fi

	printf "%b\n\n" "$TEXT" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
	printf %b "$TEXT\n" | sxmo_modem.sh formatbody >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"

	if [ "$MESSAGE_TYPE" = "Received" ]; then
	if [ "$MESSAGE_TYPE" = "recv" ]; then
		FROM_NAME="$(sxmo_contacts.sh --name "$FROM_NUM")"
		[ "$FROM_NAME" = "???" ] && FROM_NAME="$FROM_NUM"
		if [ -z "$SXMO_DISABLE_SMS_NOTIFS" ]; then
			[ -n "$OPEN_ATTACHMENTS_CMD" ] && TEXT="$icon_att $TEXT"
diff --git a/scripts/modem/sxmo_modem.sh b/scripts/modem/sxmo_modem.sh
index b268548..218ba1d 100644
--- a/scripts/modem/sxmo_modem.sh
+++ b/scripts/modem/sxmo_modem.sh
@@ -33,6 +33,40 @@ lookupnumberfromcallid() {
		tr -d ' '
}

_gen_id(){
	cat /dev/random | tr -dc "[:alnum:]" | head -c 15
}

formatheader(){
	TIME="$1"
	shift
	DIRECTION="$1"
	shift
	TYPE="$1"
	shift
	SENDER="$1"
	shift
	RECIPIENTS=""
	while [ "$#" != "0" ]; do
		RECIPIENTS="$RECIPIENTS	$1"
		shift
	done
	RECIPIENTS="$(echo "$RECIPIENTS" | sed 's/^\t//')"
	printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$TIME" "$DIRECTION" "$TYPE" "$(_gen_id)" "$SENDER" "$RECIPIENTS"
}

formatbody(){
	while IFS= read -r line; do
		printf " %b\n" "$line"
	done
	printf "\n"
}

formattext(){
	formatheader "$@"
	formatbody
}

checkforfinishedcalls() {
	#find all finished calls
	for FINISHEDCALLID in $(
@@ -140,11 +174,11 @@ checkforincomingcalls() {
}

checkfornewtexts() {
	TEXTIDS="$(
		mmcli -m any --messaging-list-sms |
		grep -Eo '/SMS/[0-9]+ \(received\)' |
		grep -Eo '[0-9]+'
	)"
TEXTIDS="$(
mmcli -m any --messaging-list-sms |
grep -Eo '/SMS/[0-9]+ \(received\)' |
grep -Eo '[0-9]+'
)"
	echo "$TEXTIDS" | grep -v . && return

	# Loop each textid received and read out the data into appropriate logfile
@@ -205,8 +239,12 @@ checkfornewtexts() {
		# the sms wap number?
		if cut -f1 "$SXMO_BLOCKFILE" 2>/dev/null | grep -q "^$NUM$"; then
			mkdir -p "$SXMO_BLOCKDIR/$NUM"
			if [ ! -e "$SXMO_BLOCKDIR/$NUM/sms.txt" ]; then
				echo "# sms_log_version: 1" > "$SXMO_BLOCKDIR/$NUM/sms.txt"
			fi
			stderr "BLOCKED text from number: $NUM (TEXTID: $TEXTID)"
			printf %b "Received from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
			printf %b "$(echo "$TEXT" | formattext "$TIME" Recv SMS "" "$NUM")" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
			# printf %b "Received from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
			printf %b "$TIME\trecv_txt\t$NUM\t${#TEXT} chars\n" >> "$SXMO_BLOCKDIR/modemlog.tsv"
			mmcli -m any --messaging-delete-sms="$TEXTID"
			continue
@@ -234,9 +272,13 @@ checkfornewtexts() {
		fi

		mkdir -p "$SXMO_LOGDIR/$NUM"
		if [ ! -e "$SXMO_LOGDIR/$NUM/sms.txt" ]; then
			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$NUM/sms.txt"
		fi
		stderr "Text from number: $NUM (TEXTID: $TEXTID)"
		printf %b "Received SMS from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_LOGDIR/$NUM/sms.txt"
		printf %b "$TIME\trecv_txt\t$NUM\t${#TEXT} chars\n" >> "$SXMO_LOGDIR/modemlog.tsv"
		printf %b "$(echo "$TEXT" | formattext "$TIME" Recv SMS "" "$NUM")" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
		# printf %b "Received SMS from $NUM at $TIME:\n$TEXT\n\n" >> "$SXMO_LOGDIR/$NUM/sms.txt"
		mmcli -m any --messaging-delete-sms="$TEXTID"
		CONTACTNAME=$(sxmo_contacts.sh --name "$NUM")
		[ "$CONTACTNAME" = "???" ] && CONTACTNAME="$NUM"
diff --git a/scripts/modem/sxmo_vvm.sh b/scripts/modem/sxmo_vvm.sh
index 7b82f49..4d8c636 100644
--- a/scripts/modem/sxmo_vvm.sh
+++ b/scripts/modem/sxmo_vvm.sh
@@ -17,9 +17,6 @@ processvvm() {
	VVM_ID="$3" # unique id assigned to voice mail from vvmd
	VVM_ATTACHMENT="$4" # full path + filename of amr file
	VVM_FILE="$SXMO_LOGDIR/$VVM_SENDER/attachments/$(basename "$VVM_ATTACHMENT")"
	VVM_SENDER_NAME="$(sxmo_contacts.sh --name "$VVM_SENDER")"
	[ "$VVM_SENDER_NAME" = "???" ] && VVM_SENDER_NAME="$VVM_SENDER"

	mkdir -p "$SXMO_LOGDIR/$VVM_SENDER/attachments"

	printf "%s\trecv_vvm\t%s\t%s\n" "$VVM_DATE" "$VVM_SENDER" "$VVM_ID" >> "$SXMO_LOGDIR/modemlog.tsv"
@@ -31,9 +28,16 @@ processvvm() {
		exit 1
	fi

	printf "Received Voice Mail from %s at %s:\n%s %s\n\n" "$VVM_SENDER_NAME" "$VVM_DATE" "$icon_att" "$(basename "$VVM_FILE")" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
	if [ ! -e "$SXMO_LOGDIR/$VVM_SENDER/sms.txt" ]; then
		echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
	fi
	sxmo_modem.sh formatheader "$VVM_DATE" recv "VVM" "" "$VVM_SENDER" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
	printf "%s %s\n\n" "$icon_att" "$(basename "$VVM_FILE")" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"

	if [ -z "$SXMO_DISABLE_SMS_NOTIFS" ]; then
		VVM_SENDER_NAME="$(sxmo_contacts.sh --name "$VVM_SENDER")"
		[ "$VVM_SENDER_NAME" = "???" ] && VVM_SENDER_NAME="$VVM_SENDER"

		sxmo_notificationwrite.sh \
			random \
			"sxmo_open.sh '$VVM_FILE'" \
-- 
2.35.1

[PATCH sxmo-utils 2/2] new text log format - migration and display

Details
Message ID
<20220302131203.374082-2-bruno.dupuis@protonmail.com>
In-Reply-To
<20220302131203.374082-1-bruno.dupuis@protonmail.com> (view parent)
DKIM signature
missing
Download raw message
Patch: +518 -11
Migration script and tailtextlog hook for the new sms format.

This is WIP...

Notes:

1. The migration script allows rollbacks, but it's not very useful as
   sxmo_modem* will use the new format anyway. They should be changed
   to detect the format of the file and save texts accordingly.

2. The tailtextlog hook is not as pretty as my previous experiments. They
   are broken when there is wide characters in the message (fail to
   right-justify). 

---
 .build.yml                                    |   2 +-
 .../default_hooks/sxmo_hook_tailtextlog.sh    | 308 +++++++++++++++++-
 migrations/1.9.0.1.sh                         | 207 ++++++++++++
 scripts/modem/sxmo_mms.sh                     |   4 +-
 scripts/modem/sxmo_modem.sh                   |   6 +-
 scripts/modem/sxmo_vvm.sh                     |   2 +-
 6 files changed, 518 insertions(+), 11 deletions(-)
 create mode 100644 migrations/1.9.0.1.sh

diff --git a/.build.yml b/.build.yml
index 10245f3..d206f30 100644
--- a/.build.yml
+++ b/.build.yml
@@ -8,7 +8,7 @@ sources:
tasks:
  - shellcheck: |
      cd sxmo-utils
      shellcheck -x scripts/*/*.sh
      shellcheck -x scripts/*/*.sh -x migrations/*.sh
  - shellspec: |
      cd sxmo-utils
      shellspec
diff --git a/configs/default_hooks/sxmo_hook_tailtextlog.sh b/configs/default_hooks/sxmo_hook_tailtextlog.sh
index 9a2b760..27ebcca 100755
--- a/configs/default_hooks/sxmo_hook_tailtextlog.sh
+++ b/configs/default_hooks/sxmo_hook_tailtextlog.sh
@@ -3,10 +3,310 @@
# This hook displays the sms log for a numbers passed as $1

. sxmo_common.sh
NUMBER="$1"
CONTACTNAME="$(sxmo_contacts.sh | grep ": ${NUMBER}$" | cut -d: -f1)"
[ "???" = "$CONTACTNAME" ] && CONTACTNAME="$CONTACTNAME ($NUMBER)"

TERMNAME="$NUMBER SMS" sxmo_terminal.sh sh -c "tail -n9999 -f \"$SXMO_LOGDIR/$NUMBER/sms.txt\" | sed \"s|$NUMBER|$CONTACTNAME|g\""
SXMO_TEXT_BACKLOG="${SXMO_TEXT_BACKLOG:-500}"

num2contact(){
	contact="$(sxmo_contacts.sh --all | grep -e ": $@$" | cut -d ":" -f 1)"
	if [ -z "$contact" ]; then
		contact="$@"
	fi
	return contact
}

formatlog(){
	NUMBER="$1"
	SMSLOG="$SXMO_LOGDIR/$NUMBER/sms.txt"
	if [ ! -e "$SMSLOG" ]; then
		mkdir -p "$(dirname "$SMSLOG")"
		echo "# text_log_version: 1" > "$SMSLOG"
	fi
	HEADER_PLACEHOLDER="$(cat /dev/random | head -c 50 | base64)"

	# first, we remove headers (replaced by HEADER_PLACEHOLDER). We `tac`
	# the file to count the number of messages we display
	msgs="$(mktemp)"
	headers="$(mktemp)"
	trap "rm -f $msgs" INT TERM EXIT
	trap "rm -f $headers" INT TERM EXIT
	tac "$SMSLOG" | awk \
		-v headers="$headers" \
		-v BACKLOG="$SXMO_TEXT_BACKLOG" \
		-v HEADER_PLACEHOLDER="$HEADER_PLACEHOLDER" \
		'
		# message content
		/^ .*/ {
			print $0
			next
		}
		# comment... ignore
		/^#.*/ {
			next
		}
		# header
		{
			print hd > headers
			print HEADER_PLACEHOLDER
			BACKLOG--
			if(BACKLOG == 0) exit
		}

	' | tac > "$msgs"

	# sort the messages based on date (not TZ aware, yet), format headers and
	# print content.
	tac "$headers" | \
	awk \
		-v msgs_file="$msgs" \
		-v NUM2CONTACT="$SCRIPT num2contact " \
		-v HEADER_PLACEHOLDER="$HEADER_PLACEHOLDER" \
		'
		# quicksort
		function less_than(left, right) {
			return "" left <= "" right
		}
		function quicksort(data, left, right, i, last){
			if (left >= right)
				return
			quicksort_swap(data, left, int((left + right) / 2))
			last = left
			for (i = left + 1; i <= right; i++)
				if (less_than(data[i], data[left]))
					quicksort_swap(data, ++last, i)
			quicksort_swap(data, left, last)
			quicksort(data, left, last - 1)
			quicksort(data, last + 1, right)
		}
		function quicksort_swap(data, i, j, temp){
			temp = data[i]
			data[i] = data[j]
			data[j] = temp
		}

		# memoized contact name resolution
		function num2contact(num,  contact){
			if(! contacts[num]){
				NUM2CONTACT num | getline contact
				contacts[num] = contact
				close(NUM2CONTACT num)
			}
			return contacts[num]
		}
		function format_header(hd,  parts, date, dir, msg_type, from, postfix){
			split(hd, parts, "\t")
			date = parts[1]
			from = ""
			if (parts[2] == "sent") {
				dir = "<<<< Sent "
				postfix = " <<<<"
			}
			else {
				dir = ">>>> Received "
				postfix = " >>>>"
				if(parts[7]) {
					# multiple recipients -> group text -> print sender s name
					from = " from " num2contact parts[5]
				}
			}
			msg_type = dir[3]
			printf("%s%s%s%s", dir, msg_type, from, postfix)
		}
		
		BEGIN {
			split("", headers)
			split("", header_keys)
			split("", messages)
			headerindex = 0
			msgindex = 0
			# contact names cache
			split("", contacts)
		}
		FILENAME == "-" {
			headerindex++
			headers[headerindex] = $0
			header_keys[$0] = headerindex
			next
		}
		$0 == HEADER_PLACEHOLDER {
			msgindex++
			next
		}
		{
			if (!messages[msgindex]) messages[msgindex] = $0
			else messages[msgindex] = messages[msgindex] "\n" $0
		}
		END {
			for (key in header_keys) { keys[n++]=key }
			quicksort(keys, 0, n-1)
			for (i = 0; i < n; i++) {
				idx = header_keys[keys[i]]
				print format_header(headers[idx])
				print messages[idx]
			}
		}
		' - "$msgs" #|\
	exit

	 # format headers and align messages depending on the sender
	 # (right = me, left = other)
	 awk \
		-v WIDTH="$WIDTH" \
		-v TODAY="$TODAY" \
		-v YESTERDAY="$YESTERDAY" \
		-v TWO_DAYS="$TWO_DAYS" \
		-v MAX_WIDTH="$MAX_WIDTH" \
		-v CONTACTNAME="$CONTACTNAME" \
		-v MSG_COUNT="$(wc -l "$headers")" '
		# utils
		function repeat(c, num) {
			s = ""
			for(i=0;i<num;i++) s = s c;
			return s
		}

		function min(a, b){
			if(a<b) return a
			return b
		}

		function max(a, b){
			if(a>b) return a
			return b
		}

		function print_msg(){
			if(them){
				return
			} else {
				max_length = 0
				for(ii=0;ii<length(msg);ii++){
					max_length = max(max_length, length(msg[ii]))
				}
				max_length = min(max_length, MAX_WIDTH)
				padding = repeat(" ", WIDTH - max_length)
				for(ii=0;ii<length(msg);ii++){
					# print "@" msg[ii] "@"
					print padding msg[ii]
				}
				split("", msg)
			}
		}

		# format the date and add human day names ("today", "yesterday", "2 days ago")
		function get_human_date(datetime){
			date = gensub("^.*( [0-9-]+)T.*", "\\1", "g", datetime)
			if(date ~ "^ " TODAY "$") {
				return ""
			}
			if(date ~ "^ " YESTERDAY "$") {
				return " yesterday at"
			}
			if(date ~ "^ " TWO_DAYS "$") {
				return " 2 days ago at"
			}
			return date
		}

		# Parse the message header to extract the time of the message.
		# Remove the timzone info
		function get_time(datetime){
			return gensub("^.* [0-9-]+T([0-9:]+).*", "\\1", "g", datetime)
		}

		BEGIN {
			them = 1
			split("", msg)
			msg_count=int(MSG_COUNT)
			timezone_re="[+-][0-9]{2}:?[0-9]{2}"
		}

		# Parse, reformat and print message headers
		/^[0-9]+\tSent SMS|MMS to .* at [0-9T:+-]+:$/{
			# dump the preceding message if needed
			print_msg()
			them=0
			title = " [" MSG_COUNT-- "]" get_human_date($0) " " get_time($0) " --"
			prefix_len=WIDTH - length(title)
			print "\033[37m" repeat("-", prefix_len) title "\033[m"
			next
		}

		/^[0-9]+\tReceived SMS|MMS from .* at [0-9T:+-]+:$/{
			print_msg()
			them=1
			title = "-- [" MSG_COUNT-- "]" get_human_date($0) " " get_time($0) " "
			postfix_len=WIDTH - length(title)
			print "\033[37m" title repeat("-", postfix_len) "\033[m"
			next
		}

		# Not a header
		{
			if(them){
				# Print early, it is left aligned
				print
				next
			} else {
				# Append the line to the message array
				# to pad when we konw the longuest line.
				msg[length(msg)] = $0
			}
		}

		END {
			# dump the last message, as no header triggerd the dump.
			print_msg()
			# add a title (the contact name)
			title = "--[ " CONTACTNAME " ]"
			postfix_len=WIDTH - length(title)
			title = "\033[34m" title repeat("-", postfix_len) "\033[m"
			printf("%s", title)
		}
	'
}

textlogloop(){
	# The loop watches the modem log file (comparing md5 sums, we could
	# use inotify for this, too).
	# On change, format the messages in a temp file, clear the term and
	# dump the tempfile.

	NUMBER="$1"
	SMSLOG="$SXMO_LOGDIR/$NUMBER/sms.txt"

	resultfile="$(mktemp)"
	trap "rm -f '$resultfile'" INT TERM EXIT

	dumplogs() {
		SUM="1234 $SMSLOG"
		while ! echo "$SUM" | md5sum -c - >/dev/null 2>&1; do
			tput civis
			TERM_LINES="$(tput lines)"
			formatlog "$NUMBER" > "$resultfile"
			printf "\033c"
			cat "$resultfile"
			SUM="$(md5sum "$SMSLOG")"
		done
	}

	dumplogs

	while inotifywait -e modify "$SMSLOG" > /dev/null 2>&1; do
		dumplogs
	done
}

if [ "1" == "$#" ]; then
	NUMBER="$1"
	if grep -qe "^# text_log_version: 1$" "$SXMO_LOGDIR/$NUMBER/sms.txt"; then
		# migrated to versioned (v1) format
		TERMNAME="$NUMBER SMS" sxmo_terminal.sh sh -c "$0 textlogloop '$NUMBER'"
	else
		# old unversioned format. Should we send a notification?
		CONTACTNAME="$(sxmo_contacts.sh | grep ": ${NUMBER}$" | cut -d: -f1)"
		[ "???" = "$CONTACTNAME" ] && CONTACTNAME="$CONTACTNAME ($NUMBER)"
		TERMNAME="$NUMBER SMS" sxmo_terminal.sh sh -c "tail -n9999 -f \"$SXMO_LOGDIR/$NUMBER/sms.txt\" | sed \"s|$NUMBER|$CONTACTNAME|g\""
	fi
else
	"$@"
fi
diff --git a/migrations/1.9.0.1.sh b/migrations/1.9.0.1.sh
new file mode 100644
index 0000000..c2f1957
--- /dev/null
+++ b/migrations/1.9.0.1.sh
@@ -0,0 +1,207 @@
#!/bin/sh

# Migrate text logs (~/.local/share/sxmo/{modem,block}/*/sms.txt) to
# the new storage version. This script creates a backup file for
# each migrated file.

# shellcheck source=scripts/core/sxmo_common.sh
. "$(which sxmo_common.sh)"

SCRIPT="$0"

contact2num (){
	contact="$1"
	if [ "$contact" = "???" ]; then
		printf "???"
	else
		num="$(sxmo_contacts.sh --all | grep -e "^$contact:" | head -1 | cut -d":" -f2- | sed 's/^\s*//')"
		if [ -z "$num" ]; then
			printf "???"
		else
			printf "%s" "$num"
		fi
	fi
}

gen_id(){
	base64 < /dev/random | tr -dc "[:alnum:]" | head -c 1500
}

get_files(){
	find "$SXMO_BLOCKDIR" "$SXMO_LOGDIR" -name sms.txt |\
		while read -r f; do
			if ! grep -qe "^# text_log_version: 1$" "$f"; then
				echo "$f"
			fi
		done
}

get_bkp_files(){
	find "$SXMO_BLOCKDIR" "$SXMO_LOGDIR" -name sms.txt.migrate.bkp
}

migratefile() {
	# set -x
	file="$1"
	CONTACTDIR="$(basename "$(dirname "$file")")"
	output="$file.migrate.tmp"
	bkp="$file.migrate.bkp"
	CONTACTS="$(echo "$CONTACTDIR" | sed 's/\+/\t+/g' | sed 's/^\t//')"

	# this logic is the same as the logic found in sxmo_mms.sh
	MYNUM="$(sxmo_contacts.sh --me)"
	if [ -z "$MYNUM" ]; then
		sxmo_log "The migration script can't infer your phone number."
		sxmo_log "This probably means you need to configure the Me contact."
		sxmo_log "We will use a fake number (+12345670000) instead."
		MYNUM="+12345670000"
	fi

	cp "$file" "$bkp"
	sxmo_log "backup file $bkp created"

	busybox awk \
	-v CONTACT2NUM="$SCRIPT contact2num " \
	-v CONTACTS="$CONTACTS" \
	-v MYNUM="$MYNUM" \
	-v GENID="$SCRIPT gen_id" \
	'
	BEGIN{
		# phone number cache
		split("", nums)

		# cache of MMS Group recipient list without the sender
		split("", num_lists)

		# Calling an external procces is very costly. We generate
		# a 1500 chars long random id at once and splice it.
		# (see get_id() definition)
		id_pool = ""

		# add a format version number.
		print "# text_log_version: 1"
	}

	function get_id(nxt){
		if (id_pool == "") {
			GENID | getline id_pool
			close(GENID)
		}
		nxt = substr(id_pool, 0, 15)
		id_pool = substr(id_pool, 16)
		return nxt
	}

	function contact2num(contact){
		if(! nums[contact]){
			CONTACT2NUM contact | getline num
			nums[contact] = num
			close(CONTACT2NUM contact)
		}
		return nums[contact]
	}

	function rm_from_list(num, lst,  key, newlst, lstarr, i){
		key = num " " lst
		if(! num_lists[key]){
			split(lst, lstarr, "\t")
			newlst = ""
			for(i in lstarr) {
				if(num != lstarr[i]){
					if(newlst == ""){
						newlst = lstarr[i]
					} else {
						newlst = newlst "\t" lstarr[i]
					}
				}
			}
			num_lists[key] = newlst
		}
		return num_lists[key]
	}

	function get_date(hd) {
		return gensub("^.* at (.*):$", "\\1", "g", hd)
	}

	# SMS
	/^Sent SMS to .* at [0-9T:+-]+:$/ {
		sender = MYNUM
		recipient = CONTACTS
		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "sent", "SMS", get_id(), sender, recipient)
		next
	}
	/^Received SMS from .* at [0-9T:+-]+:$/ {
		sender = CONTACTS
		recipient = MYNUM
		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "SMS", get_id(), sender, recipient)
		next
	}

	# 1-to-1 MMS
	/^Sent MMS from .* at [0-9T:+-]+:$/ {
		sender = MYNUM
		recipient = CONTACTS
		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "sent", "MMS", get_id(), sender, recipient)
		next
	}
	/^Received MMS from .* at [0-9T:+-]+:$/ {
		sender = CONTACTS
		recipient = MYNUM
		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "MMS", get_id(), sender, recipient)
		next
	}

	# Voice Mail
	/^Received Voice Mail from .* at [0-9T:+-]+:$/ {
		sender = CONTACTS
		recipient = MYNUM
		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "VVM", get_id(), sender, recipient)
		next
	}

	# Group MMS
	/^Sent Group MMS from .+ to .+ at [0-9T:+-]+:$/ {
		sender = MYNUM
		recipient = CONTACTS
		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "sent", "MMS", get_id(), sender, recipient)
		next
	}
	/^Received Group MMS from .* to .* at [0-9T:+-]+:$/ {
		sender = gensub("Received Group MMS from (.+) to .+ at [0-9T:+-]+:$", "\\1", "g", $0)
		sender = contact2num(sender)
		recipient = rm_from_list(sender, CONTACTS "\t" MYNUM)
		printf("%s\t%s\t%s\t%s\t%s\t%s\n", get_date($0), "recv", "MMS", get_id(), sender, recipient)
		next
	}

	# message content
	{
		print " " $0
	}
	' "$file" > "$output"
	mv "$output" "$file"
}

migrate_all(){
	sxmo_log "Migrate all sms log files."
	sxmo_log "A backup is created. In case of unexpected behaviour, revert with:"
	sxmo_log "sh $SCRIPT rollback"
	get_files | while read -r file; do
		sxmo_log "Migarting $file"
		migratefile "$file"
	done
}

rollback(){
	get_bkp_files | while read -r src; do
		target="${src%.migrate.bkp}"
		mv "$src" "$target"
	done
}

if [ "0" = "$#" ]; then
	migrate_all
else
	"$@"
fi
diff --git a/scripts/modem/sxmo_mms.sh b/scripts/modem/sxmo_mms.sh
index 5ef375f..0a54a65 100644
--- a/scripts/modem/sxmo_mms.sh
+++ b/scripts/modem/sxmo_mms.sh
@@ -197,7 +197,7 @@ processmms() {
		LOGDIRNUM="$(printf "%b\n%s\n" "$TO_NUMS" "$FROM_NUM" | grep -v "^$MYNUM$" | sort -u | grep . | xargs printf %s)"
		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
			echo "# text_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		fi
		sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$(printf %s "$TO_NUMS" | tr '\n' '\t')" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		# printf "%s Group MMS from %s to %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$TO_NAMES" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
@@ -210,7 +210,7 @@ processmms() {
		fi
		mkdir -p "$SXMO_LOGDIR/$LOGDIRNUM"
		if [ ! -e "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt" ]; then
			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
			echo "# text_log_version: 1" > "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		fi
		printf "%s\n" "$(sxmo_modem.sh formatheader "$DATE" "$MESSAGE_TYPE" "MMS" "$FROM_NUM" "$TO_NUMS")" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
		# printf "%s MMS from %s at %s:\n" "$MESSAGE_TYPE" "$FROM_NAME" "$DATE" >> "$SXMO_LOGDIR/$LOGDIRNUM/sms.txt"
diff --git a/scripts/modem/sxmo_modem.sh b/scripts/modem/sxmo_modem.sh
index 218ba1d..ac4fd91 100644
--- a/scripts/modem/sxmo_modem.sh
+++ b/scripts/modem/sxmo_modem.sh
@@ -34,7 +34,7 @@ lookupnumberfromcallid() {
}

_gen_id(){
	cat /dev/random | tr -dc "[:alnum:]" | head -c 15
	base64 < /dev/random | tr -dc "[:alnum:]" | head -c 1500
}

formatheader(){
@@ -240,7 +240,7 @@ grep -Eo '[0-9]+'
		if cut -f1 "$SXMO_BLOCKFILE" 2>/dev/null | grep -q "^$NUM$"; then
			mkdir -p "$SXMO_BLOCKDIR/$NUM"
			if [ ! -e "$SXMO_BLOCKDIR/$NUM/sms.txt" ]; then
				echo "# sms_log_version: 1" > "$SXMO_BLOCKDIR/$NUM/sms.txt"
				echo "# text_log_version: 1" > "$SXMO_BLOCKDIR/$NUM/sms.txt"
			fi
			stderr "BLOCKED text from number: $NUM (TEXTID: $TEXTID)"
			printf %b "$(echo "$TEXT" | formattext "$TIME" Recv SMS "" "$NUM")" >> "$SXMO_BLOCKDIR/$NUM/sms.txt"
@@ -273,7 +273,7 @@ grep -Eo '[0-9]+'

		mkdir -p "$SXMO_LOGDIR/$NUM"
		if [ ! -e "$SXMO_LOGDIR/$NUM/sms.txt" ]; then
			echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$NUM/sms.txt"
			echo "# text_log_version: 1" > "$SXMO_LOGDIR/$NUM/sms.txt"
		fi
		stderr "Text from number: $NUM (TEXTID: $TEXTID)"
		printf %b "$TIME\trecv_txt\t$NUM\t${#TEXT} chars\n" >> "$SXMO_LOGDIR/modemlog.tsv"
diff --git a/scripts/modem/sxmo_vvm.sh b/scripts/modem/sxmo_vvm.sh
index 4d8c636..00b1b30 100644
--- a/scripts/modem/sxmo_vvm.sh
+++ b/scripts/modem/sxmo_vvm.sh
@@ -29,7 +29,7 @@ processvvm() {
	fi

	if [ ! -e "$SXMO_LOGDIR/$VVM_SENDER/sms.txt" ]; then
		echo "# sms_log_version: 1" > "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
		echo "# text_log_version: 1" > "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
	fi
	sxmo_modem.sh formatheader "$VVM_DATE" recv "VVM" "" "$VVM_SENDER" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
	printf "%s %s\n\n" "$icon_att" "$(basename "$VVM_FILE")" >> "$SXMO_LOGDIR/$VVM_SENDER/sms.txt"
-- 
2.35.1

[sxmo-utils/patches/.build.yml] build success

builds.sr.ht <builds@sr.ht>
Details
Message ID
<CI9ETLVDAR4T.1PL8A67V2JXCS@cirno2>
In-Reply-To
<20220302131203.374082-2-bruno.dupuis@protonmail.com> (view parent)
DKIM signature
missing
Download raw message
sxmo-utils/patches/.build.yml: SUCCESS in 21s

[use new sms log format][0] from [Bruno Dupuis][1]

[0]: https://lists.sr.ht/~mil/sxmo-devel/patches/29935
[1]: bruno.dupuis@protonmail.com

✓ #707093 SUCCESS sxmo-utils/patches/.build.yml https://builds.sr.ht/~mil/job/707093

Re: [PATCH sxmo-utils 2/2] new text log format - migration and display

Details
Message ID
<CI9FOJU6TASA.182OJ51UVHVN2@uknow>
In-Reply-To
<20220302131203.374082-2-bruno.dupuis@protonmail.com> (view parent)
DKIM signature
missing
Download raw message
If you want to apply and test this, please, be very careful, it's in pre-alpha stage,
I've tested this only with dummy data on my laptop, with sxmo_* scripts stubs. It may eat your
data. At least, crate a backup of ~/.cache/share/sxmo/modem before running
sxmo_migrate.sh.
Reply to thread Export thread (mbox)