2022-07-21 20:53:21 +00:00
#!/bin/sh
# Do various clean-up tasks in matrix postgresql database
# run as cron job all 6h or so
2022-08-27 09:36:50 +00:00
# run with argument reset every month or every week
2022-07-21 20:53:21 +00:00
# URL to connect to matrix
MATRIXURL = "http://localhost:8008"
# matrix postgresql database credentials
SQLUSER = synapse
SQLPASSWD = password
SQLDB = synapse
# path to synapse_auto_compressor binary
# see https://github.com/matrix-org/rust-synapse-compress-state
SYNAPSE_COMPRESSOR_PATH = "/etc/matrix-synapse/synapse_auto_compressor"
# prefix for bridge users and rooms (hardcoded in bridge)
PREFIX = "pp"
# admin user of bridge channels
# (bridge creates him automatically, just make sure to run it at least once before running this script)
ADMINID = " @ ${ PREFIX } _admin:pixelplanet.fun "
echo "----------CLEANING UP POSTGRESQL MATRIX DATABASE------------"
cd /var/lib/postgresql
echo "--Get token for admin user"
TOKEN = ` psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c " select token from access_tokens where user_id = ' ${ ADMINID } ' and device_id = 'SQLCLEANER' limit 1; " | xargs`
if [ -z ${ TOKEN } ]
then
echo "Non exists, generating new Token..."
TOKEN = ` cat /proc/sys/kernel/random/uuid`
TOKENID = ` psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c "select max(id) + 1 from access_tokens" `
psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c " insert into access_tokens(id, user_id, token, device_id, last_validated, used) values ( ${ TOKENID } , ' ${ ADMINID } ', ' ${ TOKEN } ', 'SQLCLEANER', 1656788062940, 'f') "
fi
2022-08-27 09:36:50 +00:00
[ " ${ 1 } " = "reset" ] && {
# reset state_auto_compressor
echo "-- Reset rust-synapse-compress-state"
psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c "drop table state_compressor_state; drop table state_compressor_progress; drop table state_compressor_total_progress;"
}
# purge events in a room - just for reference - ignore this block
# TODO provide a function that purges a room and also its state_groups
#ROOM="scTbMproDsaihhGesQ:pixelplanet.fun"
#DELURL="${MATRIXURL}/_synapse/admin/v1/purge_history/!${ROOM}"
#curl --insecure -XPOST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" -d "{\"delete_local_events\": true, \"purge_up_to_ts\": 1660340843343}" ${DELURL}
# and its status check
#DELURL="${MATRIXURL}/_synapse/admin/v1/purge_history_status/MtGGeIGbsYTqdCUF"
#curl --insecure -XGET -H "Authorization: Bearer ${TOKEN}" ${DELURL}
# Remember that if you delete a room, it's state groups are still around
# https://github.com/erikjohnston/synapse-find-unreferenced-state-groups
#exit
# -----------------------------------------------------------------
2022-07-21 20:53:21 +00:00
# Disable ratelimit
echo " --Disabling ratelimit for admin user ${ ADMINID } ... "
RATEURL = " ${ MATRIXURL } /_synapse/admin/v1/users/ ${ ADMINID } /override_ratelimit "
curl --insecure -XPOST -H " Authorization: Bearer ${ TOKEN } " -H "Content-Type: application/json" -d "{\"messages_per_second\": 0}" ${ RATEURL }
ROOMS = ` psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c " select room_id from room_aliases where room_alias like '# ${ PREFIX } _%' " `
get_curl_config ( ) {
room = ${ 1 }
# Get all appsockets users from a public room that did not send any message in the past 48h
2022-08-05 17:56:03 +00:00
ASUSERS = ` psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c " select user_id from users_in_public_rooms u where room_id = ' ${ room } ' and user_id like '@ ${ PREFIX } _%:pixelplanet.fun' and user_id != ' ${ ADMINID } ' and not exists ( select from events where room_id = ' ${ room } ' and sender = u.user_id and to_timestamp(received_ts/1000) > now() - interval '4 DAYS' ) and not exists ( select from user_ips where user_id = u.user_id ) " `
2022-07-21 20:53:21 +00:00
KICKURL = " ${ MATRIXURL } /_matrix/client/v3/rooms/ ${ room } /kick "
CNT = 1
for user in ${ ASUSERS }
do
if [ ${ CNT } -eq 1 ]
then
CNT = 0
else
echo "next"
fi
echo " header=\"Authorization: Bearer ${ TOKEN } \" "
echo "header=\"Content-Type: application/json\""
echo " data=\"{\\\"user_id\\\": \\\" ${ user } \\\"}\" "
echo " url= ${ KICKURL } "
done
return ${ CNT }
}
for room in ${ ROOMS }
do
echo " --Delete event_push_actions of not logged-in users from romm ${ room } ... "
# Clean event_push_actions of not-logged-in application service users
# see https://github.com/matrix-org/synapse/issues/5569
# This command can be really slow, if it takes too long, remove the "not exists..." part and it won't care about if logged in or not
2022-08-05 17:56:03 +00:00
time psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c " delete from event_push_actions u where room_id = ' ${ room } ' and user_id like '@ ${ PREFIX } _%:pixelplanet.fun' and not exists ( select from user_ips where user_id = u.user_id and to_timestamp(last_seen/1000) > now() - interval '2 days' ) "
2022-07-21 20:53:21 +00:00
echo " --Kick out inactive users from room ${ room } ... "
get_curl_config " ${ room } " > /tmp/curlkick.tmp && curl --parallel --parallel-immediate --parallel-max 10 --config /tmp/curlkick.tmp && echo ""
rm /tmp/curlkick.tmp
done
echo "--Clean up cache_invalidation_stream_by_instance"
# see https://github.com/matrix-org/synapse/issues/8269
time psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c "delete from cache_invalidation_stream_by_instance where to_timestamp(invalidation_ts/1000) > now() - interval '1 months';"
echo "--Compress states..."
# https://github.com/matrix-org/rust-synapse-compress-state
${ SYNAPSE_COMPRESSOR_PATH } -p postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c 500 -n 100
echo "--Vaccum..."
time psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c "VACUUM FULL VERBOSE"
echo "--DONE. Current database size is..."
time psql -t postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c "SELECT pg_size_pretty( pg_database_size( 'synapse' ) )"
2022-07-27 11:49:52 +00:00
[ -n " ${ BASH_VERSION } " ] && set +H
psql postgresql://${ SQLUSER } :${ SQLPASSWD } @localhost/${ SQLDB } -c "SELECT nspname || '.' || relname AS \"relation\", pg_size_pretty(pg_total_relation_size(c.oid)) AS \"total_size\" FROM pg_class c LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace) WHERE nspname NOT IN ('pg_catalog', 'information_schema') AND c.relkind <> 'i' AND nspname !~ '^pg_toast' ORDER BY pg_total_relation_size(c.oid) DESC LIMIT 20;"
2022-08-27 09:36:50 +00:00