From 9f918ba30cf7632e5dfa724ee90f74e587fb784c Mon Sep 17 00:00:00 2001 From: sceox Date: Sat, 9 Oct 2021 11:04:48 -0700 Subject: [PATCH] refactor synapse purge and compress to run non-interactively --- Makefile | 8 ++- README.md | 22 +++++++- synapse-compress.sh | 36 ------------- synapse-purge.sh => synapse-purge-compress.sh | 50 +++++++++++++++---- 4 files changed, 63 insertions(+), 53 deletions(-) delete mode 100755 synapse-compress.sh rename synapse-purge.sh => synapse-purge-compress.sh (54%) diff --git a/Makefile b/Makefile index 851fe8a..755bdf0 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,7 @@ all: install -install: - install -m 755 synapse-compress.sh /usr/local/bin/synapse-compress.sh - install -m 755 synapse-purge.sh /usr/local/bin/synapse-purge.sh +install: synapse-purge-compress.sh + install -m 755 synapse-purge-compress.sh /usr/local/bin/synapse-purge-compress uninstall: - rm /usr/local/bin/synapse-compress.sh - rm /usr/local/bin/synapse-purge.sh + rm /usr/local/bin/synapse-purge-compress diff --git a/README.md b/README.md index 0fc504c..237bf72 100644 --- a/README.md +++ b/README.md @@ -18,5 +18,23 @@ With this repository as your working directory, run `make` as root. ## Using the scripts -The scripts can only be run interactively, as they prompt for certain -passwords and parameters. They do not parse any arguments. +### synapse-purge-compress + +The postgres database for the matrix synapse server grows big fast. +This script brings its size back down by purging old data and +compressing the state tables. + +Run as the matrix-synapse user, for example: + + sudo -u matrix-synapse synapse-purge-compress + +Run from cron: + + sudo -u matrix-synapse crontab -e + +Add a line like so: + + 0 2 1 * * /usr/local/bin/synapse-purge-compress + +NOTE: the script is still very verbose, so you may want to run it +with chronic from [moreutils](https://www.putorius.net/moreutils.html). diff --git a/synapse-compress.sh b/synapse-compress.sh deleted file mode 100755 index ba28e0b..0000000 --- a/synapse-compress.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh - -# don't bother compressing unless we will save this much: -min_compression_percent=20 - -read -p "synapse admin API access token: " token -read -p "postgres synapse user password: " db_password - -get_synapse_version () { - curl --silent --ssl -H "Authorization: Bearer $token" \ - "localhost:8008/_synapse/admin/v1/server_version" -} - -get_all_rooms () { - curl --silent -H "Authorization: Bearer $token" \ - "localhost:8008/_synapse/admin/v1/rooms" \ - | jq '.rooms[].room_id' \ - | sed 's/"//g' -} - -compress_state () { - for room_id ; do - sqlf="$HOME/$(echo $room_id | tr -c -d '[:alpha:]').sql" - repl=$(synapse-compress-state -t -o $sqlf -p \ - "host=localhost user=synapse password=${db_password} dbname=synapse" \ - -r "$room_id" | sed -n '/%/s/.*(\([0-9]*\).[0-9]*%).*/\1/p') - - if [ "$repl" -le "$((100 - $min_compression_percent))" ]; then - echo "compressing room" $room_id "..." - psql -q -U 'synapse' -f $sqlf 'synapse' - fi - rm $sqlf - done -} - -compress_state $(get_all_rooms) diff --git a/synapse-purge.sh b/synapse-purge-compress.sh similarity index 54% rename from synapse-purge.sh rename to synapse-purge-compress.sh index 72d88c8..3d44c46 100755 --- a/synapse-purge.sh +++ b/synapse-purge-compress.sh @@ -3,31 +3,43 @@ # guide: https://levans.fr/shrink-synapse-database.html # doc: https://matrix-org.github.io/synapse/develop/admin_api/purge_history_api.html +host='localhost' +port='8008' synapse_log='/var/log/matrix-synapse/homeserver.log' +homeserver_config='/etc/matrix-synapse/homeserver.yaml' -read -p "synapse admin API access_token: " token -read -p "history to keep (date range with no spaces, eg 1month, 1day): " range +# don't bother compressing unless we will save this much: +min_compression_percent=15 -# timestamps are in milliseconds since the epoch +if test -r /etc/matrix-synapse/access_token ; then + token="$(cat /etc/matrix-synapse/access_token)" + range="2months" + db_password="$(grep '^\s*password: ' $homeserver_config | awk '{print $2}')" +else + # TODO: if standard output is not a tty, exit with failure + read -p "synapse admin API access_token: " token + read -p "history to keep (date range with no spaces, eg 1month, 1day): " range +fi + +# synapse wants timestamps in milliseconds since the epoch ts_history="$(date -d-${range} +%s)000" ts_media="$(date -d-1month +%s)000" get_obsolete_rooms () { - tmpf=$(mktemp) curl --silent -H "Authorization: Bearer $token" \ - "localhost:8008/_synapse/admin/v1/rooms" \ + "${host}:${port}/_synapse/admin/v1/rooms" \ | jq '.rooms[] | select(.joined_local_members == 0) | .room_id' \ | sed 's/"//g' } get_synapse_version () { curl --silent --ssl -H "Authorization: Bearer $token" \ - "localhost:8008/_synapse/admin/v1/server_version" + "${host}:${port}/_synapse/admin/v1/server_version" } get_all_rooms () { curl --silent -H "Authorization: Bearer $token" \ - "localhost:8008/_synapse/admin/v1/rooms" \ + "${host}:${port}/_synapse/admin/v1/rooms" \ | jq '.rooms[].room_id' \ | sed 's/"//g' } @@ -37,7 +49,7 @@ purge_obsolete_rooms () { curl --silent --ssl -X POST --header "Content-Type: application/json" \ --header "Authorization: Bearer $token" \ -d "{ \"room_id\": \"$room_id\" }" \ - "localhost:8008/_synapse/admin/v1/purge_room" \ + "${host}:${port}/_synapse/admin/v1/purge_room" \ | jq done } @@ -50,7 +62,7 @@ purge_history () { --header "Content-Type: application/json" \ --header "Authorization: Bearer $token" \ -d "{ \"delete_local_events\": false, \"purge_up_to_ts\": $ts_history }" \ - "localhost:8008/_synapse/admin/v1/purge_history/${room_id}") + "${host}:${port}/_synapse/admin/v1/purge_history/${room_id}") echo $json | jq if echo $json | grep -q 'purge_id'; then echo "waiting for purge to complete..." @@ -66,10 +78,28 @@ purge_history () { purge_media_cache () { curl --silent -X POST --ssl \ --header "Authorization: Bearer $token" -d '' \ - "localhost:8008/_synapse/admin/v1/purge_media_cache?before_ts=${ts_media}" \ + "${host}:${port}/_synapse/admin/v1/purge_media_cache?before_ts=${ts_media}" \ | jq } +compress_state () { + umask 077 + for room_id ; do + sqlf="/tmp/$(echo $room_id | tr -c -d '[:alpha:]').sql" + repl=$(synapse-compress-state -t -o $sqlf -p \ + "host=${host} user=synapse password=${db_password} dbname=synapse" \ + -r "$room_id" | sed -n '/%/s/.*(\([0-9]*\).[0-9]*%).*/\1/p') + + if [ "$repl" -le "$((100 - $min_compression_percent))" ]; then + echo "compressing room" $room_id "..." + psql -q -U 'synapse' -f $sqlf 'synapse' + fi + rm $sqlf + done +} + purge_obsolete_rooms purge_history $(get_all_rooms) purge_media_cache +compress_state $(get_all_rooms) +psql -q -c "REINDEX DATABASE synapse;" -c "VACUUM FULL;" -d synapse -U synapse