add purge script
This commit is contained in:
parent
d8be801b3b
commit
e5e6d21257
17
purge/README.md
Normal file
17
purge/README.md
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Purge old meia
|
||||
|
||||
Shell script that cleans up the matrix postgresql database,
|
||||
removes push notifications that aren't needed,
|
||||
kicks out inactive users from rooms,
|
||||
cleans up states with synapse_auto_compressor and so on.
|
||||
|
||||
## Running
|
||||
|
||||
1. Set SQL credentials and URL to local matrix in the script
|
||||
2. build [synapse_auto_compressor](https://github.com/matrix-org/rust-synapse-compress-state) and set its path in the script
|
||||
3. make sure that the bridge did start at least once (it creates rooms and adds an admin user that we need)
|
||||
4. add it as a cron job all 6h or so, like:
|
||||
|
||||
```
|
||||
0 */6 * * * root /etc/matrix-synapse/matrixpurge.sh
|
||||
```
|
||||
91
purge/matrixpurge.sh
Executable file
91
purge/matrixpurge.sh
Executable file
|
|
@ -0,0 +1,91 @@
|
|||
#!/bin/sh
|
||||
# Do various clean-up tasks in matrix postgresql database
|
||||
# run as cron job all 6h or so
|
||||
|
||||
# URL to connect to matrix
|
||||
MATRIXURL="http://localhost:8008"
|
||||
# matrix postgresql database credentials
|
||||
SQLUSER=synapse
|
||||
SQLPASSWD=password
|
||||
SQLDB=synapse
|
||||
# path to synapse_auto_compressor binary
|
||||
# see https://github.com/matrix-org/rust-synapse-compress-state
|
||||
SYNAPSE_COMPRESSOR_PATH="/etc/matrix-synapse/synapse_auto_compressor"
|
||||
|
||||
|
||||
# prefix for bridge users and rooms (hardcoded in bridge)
|
||||
PREFIX="pp"
|
||||
# admin user of bridge channels
|
||||
# (bridge creates him automatically, just make sure to run it at least once before running this script)
|
||||
ADMINID="@${PREFIX}_admin:pixelplanet.fun"
|
||||
|
||||
echo "----------CLEANING UP POSTGRESQL MATRIX DATABASE------------"
|
||||
|
||||
cd /var/lib/postgresql
|
||||
|
||||
echo "--Get token for admin user"
|
||||
|
||||
TOKEN=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select token from access_tokens where user_id = '${ADMINID}' and device_id = 'SQLCLEANER' limit 1;" | xargs`
|
||||
if [ -z ${TOKEN} ]
|
||||
then
|
||||
echo "Non exists, generating new Token..."
|
||||
TOKEN=`cat /proc/sys/kernel/random/uuid`
|
||||
TOKENID=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select max(id) + 1 from access_tokens"`
|
||||
psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "insert into access_tokens(id, user_id, token, device_id, last_validated, used) values (${TOKENID}, '${ADMINID}', '${TOKEN}', 'SQLCLEANER', 1656788062940, 'f')"
|
||||
fi
|
||||
|
||||
# Disable ratelimit
|
||||
echo "--Disabling ratelimit for admin user ${ADMINID}..."
|
||||
RATEURL="${MATRIXURL}/_synapse/admin/v1/users/${ADMINID}/override_ratelimit"
|
||||
curl --insecure -XPOST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" -d "{\"messages_per_second\": 0}" ${RATEURL}
|
||||
|
||||
ROOMS=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select room_id from room_aliases where room_alias like '#${PREFIX}_%'"`
|
||||
|
||||
|
||||
get_curl_config () {
|
||||
room=${1}
|
||||
# Get all appsockets users from a public room that did not send any message in the past 48h
|
||||
ASUSERS=`psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "select user_id from users_in_public_rooms u where room_id = '${room}' and user_id like '@${PREFIX}_%:pixelplanet.fun' and user_id != '${ADMINID}' and not exists ( select from events where room_id = '${room}' and sender = u.user_id and to_timestamp(received_ts/1000) > now() - interval '48 HOURS' ) and not exists ( select from user_ips where user_id = u.user_id )"`
|
||||
KICKURL="${MATRIXURL}/_matrix/client/v3/rooms/${room}/kick"
|
||||
CNT=1
|
||||
for user in ${ASUSERS}
|
||||
do
|
||||
if [ ${CNT} -eq 1 ]
|
||||
then
|
||||
CNT=0
|
||||
else
|
||||
echo "next"
|
||||
fi
|
||||
echo "header=\"Authorization: Bearer ${TOKEN}\""
|
||||
echo "header=\"Content-Type: application/json\""
|
||||
echo "data=\"{\\\"user_id\\\": \\\"${user}\\\"}\""
|
||||
echo "url=${KICKURL}"
|
||||
done
|
||||
return ${CNT}
|
||||
}
|
||||
|
||||
|
||||
for room in ${ROOMS}
|
||||
do
|
||||
echo "--Delete event_push_actions of not logged-in users from romm ${room}..."
|
||||
# Clean event_push_actions of not-logged-in application service users
|
||||
# see https://github.com/matrix-org/synapse/issues/5569
|
||||
# This command can be really slow, if it takes too long, remove the "not exists..." part and it won't care about if logged in or not
|
||||
time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "delete from event_push_actions u where room_id = '${room}' and user_id like '@${PREFIX}_%:pixelplanet.fun' and user_id != '${ADMINID}' and not exists ( select from user_ips where user_id = u.user_id and to_timestamp(last_seen/1000) > now() - interval '4 weeks' )"
|
||||
echo "--Kick out inactive users from room ${room}..."
|
||||
get_curl_config "${room}" > /tmp/curlkick.tmp && curl --parallel --parallel-immediate --parallel-max 10 --config /tmp/curlkick.tmp && echo ""
|
||||
rm /tmp/curlkick.tmp
|
||||
done
|
||||
|
||||
echo "--Clean up cache_invalidation_stream_by_instance"
|
||||
# see https://github.com/matrix-org/synapse/issues/8269
|
||||
time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "delete from cache_invalidation_stream_by_instance where to_timestamp(invalidation_ts/1000) > now() - interval '1 months';"
|
||||
echo "--Compress states..."
|
||||
# https://github.com/matrix-org/rust-synapse-compress-state
|
||||
${SYNAPSE_COMPRESSOR_PATH} -p postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c 500 -n 100
|
||||
echo "--Vaccum..."
|
||||
time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "VACUUM FULL VERBOSE"
|
||||
echo "--DONE. Current database size is..."
|
||||
time psql -t postgresql://${SQLUSER}:${SQLPASSWD}@localhost/${SQLDB} -c "SELECT pg_size_pretty( pg_database_size( 'synapse' ) )"
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user