From e5e6d2125767169cb19d490472e8ad98206e6773 Mon Sep 17 00:00:00 2001 From: HF Date: Thu, 21 Jul 2022 22:53:21 +0200 Subject: [PATCH] add purge script --- purge/README.md | 17 +++++++++ purge/matrixpurge.sh | 91 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 purge/README.md create mode 100755 purge/matrixpurge.sh diff --git a/purge/README.md b/purge/README.md new file mode 100644 index 0000000..a928953 --- /dev/null +++ b/purge/README.md @@ -0,0 +1,17 @@ +# Purge old meia + +Shell script that cleans up the matrix postgresql database, +removes push notifications that aren't needed, +kicks out inactive users from rooms, +cleans up states with synapse_auto_compressor and so on. + +## Running + +1. Set SQL credentials and URL to local matrix in the script +2. build [synapse_auto_compressor](https://github.com/matrix-org/rust-synapse-compress-state) and set its path in the script +3. make sure that the bridge did start at least once (it creates rooms and adds an admin user that we need) +4. add it as a cron job all 6h or so, like: + +``` +0 */6 * * * root /etc/matrix-synapse/matrixpurge.sh +``` diff --git a/purge/matrixpurge.sh b/purge/matrixpurge.sh new file mode 100755 index 0000000..14e2b8f --- /dev/null +++ b/purge/matrixpurge.sh @@ -0,0 +1,91 @@ +#!/bin/sh +# Do various clean-up tasks in matrix postgresql database +# run as cron job all 6h or so + +# URL to connect to matrix +MATRIXURL="http://localhost:8008" +# matrix postgresql database credentials +SQLUSER=synapse +SQLPASSWD=password +SQLDB=synapse +# path to synapse_auto_compressor binary +# see https://github.com/matrix-org/rust-synapse-compress-state +SYNAPSE_COMPRESSOR_PATH="/etc/matrix-synapse/synapse_auto_compressor" + + +# prefix for bridge users and rooms (hardcoded in bridge) +PREFIX="pp" +# admin user of bridge channels +# (bridge creates him automatically, just make sure to run it at least once before running this script) +ADMINID="@${PREFIX}_admin:pixelplanet.fun" + +echo "----------CLEANING UP POSTGRESQL MATRIX DATABASE------------" + +cd /var/lib/postgresql + +echo "--Get token for admin user" + +TOKEN=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select token from access_tokens where user_id = '${ADMINID}' and device_id = 'SQLCLEANER' limit 1;" | xargs` +if [ -z ${TOKEN} ] +then + echo "Non exists, generating new Token..." + TOKEN=`cat /proc/sys/kernel/random/uuid` + TOKENID=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select max(id) + 1 from access_tokens"` + psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "insert into access_tokens(id, user_id, token, device_id, last_validated, used) values (${TOKENID}, '${ADMINID}', '${TOKEN}', 'SQLCLEANER', 1656788062940, 'f')" +fi + +# Disable ratelimit +echo "--Disabling ratelimit for admin user ${ADMINID}..." +RATEURL="${MATRIXURL}/_synapse/admin/v1/users/${ADMINID}/override_ratelimit" +curl --insecure -XPOST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" -d "{\"messages_per_second\": 0}" ${RATEURL} + +ROOMS=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select room_id from room_aliases where room_alias like '#${PREFIX}_%'"` + + +get_curl_config () { + room=${1} + # Get all appsockets users from a public room that did not send any message in the past 48h + ASUSERS=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select user_id from users_in_public_rooms u where room_id = '${room}' and user_id like '@${PREFIX}_%:pixelplanet.fun' and user_id != '${ADMINID}' and not exists ( select from events where room_id = '${room}' and sender = u.user_id and to_timestamp(received_ts/1000) > now() - interval '48 HOURS' ) and not exists ( select from user_ips where user_id = u.user_id )"` + KICKURL="${MATRIXURL}/_matrix/client/v3/rooms/${room}/kick" + CNT=1 + for user in ${ASUSERS} + do + if [ ${CNT} -eq 1 ] + then + CNT=0 + else + echo "next" + fi + echo "header=\"Authorization: Bearer ${TOKEN}\"" + echo "header=\"Content-Type: application/json\"" + echo "data=\"{\\\"user_id\\\": \\\"${user}\\\"}\"" + echo "url=${KICKURL}" + done + return ${CNT} +} + + +for room in ${ROOMS} +do + echo "--Delete event_push_actions of not logged-in users from romm ${room}..." + # Clean event_push_actions of not-logged-in application service users + # see https://github.com/matrix-org/synapse/issues/5569 + # This command can be really slow, if it takes too long, remove the "not exists..." part and it won't care about if logged in or not + time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "delete from event_push_actions u where room_id = '${room}' and user_id like '@${PREFIX}_%:pixelplanet.fun' and user_id != '${ADMINID}' and not exists ( select from user_ips where user_id = u.user_id and to_timestamp(last_seen/1000) > now() - interval '4 weeks' )" + echo "--Kick out inactive users from room ${room}..." + get_curl_config "${room}" > /tmp/curlkick.tmp && curl --parallel --parallel-immediate --parallel-max 10 --config /tmp/curlkick.tmp && echo "" + rm /tmp/curlkick.tmp +done + +echo "--Clean up cache_invalidation_stream_by_instance" +# see https://github.com/matrix-org/synapse/issues/8269 +time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "delete from cache_invalidation_stream_by_instance where to_timestamp(invalidation_ts/1000) > now() - interval '1 months';" +echo "--Compress states..." +# https://github.com/matrix-org/rust-synapse-compress-state +${SYNAPSE_COMPRESSOR_PATH} -p postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c 500 -n 100 +echo "--Vaccum..." +time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "VACUUM FULL VERBOSE" +echo "--DONE. Current database size is..." +time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "SELECT pg_size_pretty( pg_database_size( 'synapse' ) )" + +