refactor synapse purge and compress to run non-interactively
This commit is contained in:
sceox
parent
7b6253e11e
commit
9f918ba30c
8
Makefile
8
Makefile
@ -1,9 +1,7 @@
|
||||
all: install
|
||||
|
||||
install:
|
||||
install -m 755 synapse-compress.sh /usr/local/bin/synapse-compress.sh
|
||||
install -m 755 synapse-purge.sh /usr/local/bin/synapse-purge.sh
|
||||
install: synapse-purge-compress.sh
|
||||
install -m 755 synapse-purge-compress.sh /usr/local/bin/synapse-purge-compress
|
||||
|
||||
uninstall:
|
||||
rm /usr/local/bin/synapse-compress.sh
|
||||
rm /usr/local/bin/synapse-purge.sh
|
||||
rm /usr/local/bin/synapse-purge-compress
|
||||
|
||||
22
README.md
22
README.md
@ -18,5 +18,23 @@ With this repository as your working directory, run `make` as root.
|
||||
|
||||
## Using the scripts
|
||||
|
||||
The scripts can only be run interactively, as they prompt for certain
|
||||
passwords and parameters. They do not parse any arguments.
|
||||
### synapse-purge-compress
|
||||
|
||||
The postgres database for the matrix synapse server grows big fast.
|
||||
This script brings its size back down by purging old data and
|
||||
compressing the state tables.
|
||||
|
||||
Run as the matrix-synapse user, for example:
|
||||
|
||||
sudo -u matrix-synapse synapse-purge-compress
|
||||
|
||||
Run from cron:
|
||||
|
||||
sudo -u matrix-synapse crontab -e
|
||||
|
||||
Add a line like so:
|
||||
|
||||
0 2 1 * * /usr/local/bin/synapse-purge-compress
|
||||
|
||||
NOTE: the script is still very verbose, so you may want to run it
|
||||
with chronic from [moreutils](https://www.putorius.net/moreutils.html).
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
# don't bother compressing unless we will save this much:
|
||||
min_compression_percent=20
|
||||
|
||||
read -p "synapse admin API access token: " token
|
||||
read -p "postgres synapse user password: " db_password
|
||||
|
||||
get_synapse_version () {
|
||||
curl --silent --ssl -H "Authorization: Bearer $token" \
|
||||
"localhost:8008/_synapse/admin/v1/server_version"
|
||||
}
|
||||
|
||||
get_all_rooms () {
|
||||
curl --silent -H "Authorization: Bearer $token" \
|
||||
"localhost:8008/_synapse/admin/v1/rooms" \
|
||||
| jq '.rooms[].room_id' \
|
||||
| sed 's/"//g'
|
||||
}
|
||||
|
||||
compress_state () {
|
||||
for room_id ; do
|
||||
sqlf="$HOME/$(echo $room_id | tr -c -d '[:alpha:]').sql"
|
||||
repl=$(synapse-compress-state -t -o $sqlf -p \
|
||||
"host=localhost user=synapse password=${db_password} dbname=synapse" \
|
||||
-r "$room_id" | sed -n '/%/s/.*(\([0-9]*\).[0-9]*%).*/\1/p')
|
||||
|
||||
if [ "$repl" -le "$((100 - $min_compression_percent))" ]; then
|
||||
echo "compressing room" $room_id "..."
|
||||
psql -q -U 'synapse' -f $sqlf 'synapse'
|
||||
fi
|
||||
rm $sqlf
|
||||
done
|
||||
}
|
||||
|
||||
compress_state $(get_all_rooms)
|
||||
@ -3,31 +3,43 @@
|
||||
# guide: https://levans.fr/shrink-synapse-database.html
|
||||
# doc: https://matrix-org.github.io/synapse/develop/admin_api/purge_history_api.html
|
||||
|
||||
host='localhost'
|
||||
port='8008'
|
||||
synapse_log='/var/log/matrix-synapse/homeserver.log'
|
||||
homeserver_config='/etc/matrix-synapse/homeserver.yaml'
|
||||
|
||||
read -p "synapse admin API access_token: " token
|
||||
read -p "history to keep (date range with no spaces, eg 1month, 1day): " range
|
||||
# don't bother compressing unless we will save this much:
|
||||
min_compression_percent=15
|
||||
|
||||
# timestamps are in milliseconds since the epoch
|
||||
if test -r /etc/matrix-synapse/access_token ; then
|
||||
token="$(cat /etc/matrix-synapse/access_token)"
|
||||
range="2months"
|
||||
db_password="$(grep '^\s*password: ' $homeserver_config | awk '{print $2}')"
|
||||
else
|
||||
# TODO: if standard output is not a tty, exit with failure
|
||||
read -p "synapse admin API access_token: " token
|
||||
read -p "history to keep (date range with no spaces, eg 1month, 1day): " range
|
||||
fi
|
||||
|
||||
# synapse wants timestamps in milliseconds since the epoch
|
||||
ts_history="$(date -d-${range} +%s)000"
|
||||
ts_media="$(date -d-1month +%s)000"
|
||||
|
||||
get_obsolete_rooms () {
|
||||
tmpf=$(mktemp)
|
||||
curl --silent -H "Authorization: Bearer $token" \
|
||||
"localhost:8008/_synapse/admin/v1/rooms" \
|
||||
"${host}:${port}/_synapse/admin/v1/rooms" \
|
||||
| jq '.rooms[] | select(.joined_local_members == 0) | .room_id' \
|
||||
| sed 's/"//g'
|
||||
}
|
||||
|
||||
get_synapse_version () {
|
||||
curl --silent --ssl -H "Authorization: Bearer $token" \
|
||||
"localhost:8008/_synapse/admin/v1/server_version"
|
||||
"${host}:${port}/_synapse/admin/v1/server_version"
|
||||
}
|
||||
|
||||
get_all_rooms () {
|
||||
curl --silent -H "Authorization: Bearer $token" \
|
||||
"localhost:8008/_synapse/admin/v1/rooms" \
|
||||
"${host}:${port}/_synapse/admin/v1/rooms" \
|
||||
| jq '.rooms[].room_id' \
|
||||
| sed 's/"//g'
|
||||
}
|
||||
@ -37,7 +49,7 @@ purge_obsolete_rooms () {
|
||||
curl --silent --ssl -X POST --header "Content-Type: application/json" \
|
||||
--header "Authorization: Bearer $token" \
|
||||
-d "{ \"room_id\": \"$room_id\" }" \
|
||||
"localhost:8008/_synapse/admin/v1/purge_room" \
|
||||
"${host}:${port}/_synapse/admin/v1/purge_room" \
|
||||
| jq
|
||||
done
|
||||
}
|
||||
@ -50,7 +62,7 @@ purge_history () {
|
||||
--header "Content-Type: application/json" \
|
||||
--header "Authorization: Bearer $token" \
|
||||
-d "{ \"delete_local_events\": false, \"purge_up_to_ts\": $ts_history }" \
|
||||
"localhost:8008/_synapse/admin/v1/purge_history/${room_id}")
|
||||
"${host}:${port}/_synapse/admin/v1/purge_history/${room_id}")
|
||||
echo $json | jq
|
||||
if echo $json | grep -q 'purge_id'; then
|
||||
echo "waiting for purge to complete..."
|
||||
@ -66,10 +78,28 @@ purge_history () {
|
||||
purge_media_cache () {
|
||||
curl --silent -X POST --ssl \
|
||||
--header "Authorization: Bearer $token" -d '' \
|
||||
"localhost:8008/_synapse/admin/v1/purge_media_cache?before_ts=${ts_media}" \
|
||||
"${host}:${port}/_synapse/admin/v1/purge_media_cache?before_ts=${ts_media}" \
|
||||
| jq
|
||||
}
|
||||
|
||||
compress_state () {
|
||||
umask 077
|
||||
for room_id ; do
|
||||
sqlf="/tmp/$(echo $room_id | tr -c -d '[:alpha:]').sql"
|
||||
repl=$(synapse-compress-state -t -o $sqlf -p \
|
||||
"host=${host} user=synapse password=${db_password} dbname=synapse" \
|
||||
-r "$room_id" | sed -n '/%/s/.*(\([0-9]*\).[0-9]*%).*/\1/p')
|
||||
|
||||
if [ "$repl" -le "$((100 - $min_compression_percent))" ]; then
|
||||
echo "compressing room" $room_id "..."
|
||||
psql -q -U 'synapse' -f $sqlf 'synapse'
|
||||
fi
|
||||
rm $sqlf
|
||||
done
|
||||
}
|
||||
|
||||
purge_obsolete_rooms
|
||||
purge_history $(get_all_rooms)
|
||||
purge_media_cache
|
||||
compress_state $(get_all_rooms)
|
||||
psql -q -c "REINDEX DATABASE synapse;" -c "VACUUM FULL;" -d synapse -U synapse
|
||||
Reference in New Issue
Block a user