From be034057e096b2e56214c9a03302560b7b92018e Mon Sep 17 00:00:00 2001 From: Paul Trowbridge Date: Thu, 12 Jan 2023 18:01:26 -0500 Subject: [PATCH] vault backup: 2023-01-12 18:01:26 --- postgres.md | 280 ++++++++++++++++++++++ postgres/aggregate.md | 49 ---- postgres/concat_distinct_array_values.sql | 27 --- postgres/descriptions.sql | 28 --- postgres/foreign_data_wrapper.sql | 14 -- postgres/install.sh | 4 - postgres/jsonb_arr_aggcd.sql | 27 --- postgres/jsonb_array_add_distinct.sql | 23 -- postgres/jsonb_array_string_agg.sql | 20 -- postgres/move_data_directory.md | 14 -- postgres/out_of_memory.md | 12 - postgres/postgis.md | 3 - postgres/postgres.md | 34 --- postgres/psql.md | 1 - postgres/user.sql | 39 --- 15 files changed, 280 insertions(+), 295 deletions(-) create mode 100644 postgres.md delete mode 100644 postgres/aggregate.md delete mode 100644 postgres/concat_distinct_array_values.sql delete mode 100644 postgres/descriptions.sql delete mode 100644 postgres/foreign_data_wrapper.sql delete mode 100644 postgres/install.sh delete mode 100644 postgres/jsonb_arr_aggcd.sql delete mode 100644 postgres/jsonb_array_add_distinct.sql delete mode 100644 postgres/jsonb_array_string_agg.sql delete mode 100644 postgres/move_data_directory.md delete mode 100644 postgres/out_of_memory.md delete mode 100644 postgres/postgis.md delete mode 100644 postgres/postgres.md delete mode 100644 postgres/psql.md delete mode 100644 postgres/user.sql diff --git a/postgres.md b/postgres.md new file mode 100644 index 0000000..166becb --- /dev/null +++ b/postgres.md @@ -0,0 +1,280 @@ +Install +========================================================= + +[PostgreSQL: Linux downloads (Ubuntu)](https://www.postgresql.org/download/linux/ubuntu/) +``` +# Create the file repository configuration: +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + +# Import the repository signing key: +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + +# Update the package lists: +sudo apt-get update + +# Install the latest version of PostgreSQL. +# If you want a specific version, use 'postgresql-12' or similar instead of 'postgresql': +sudo apt-get -y install postgresql +``` + + + +SSPI +======================================================== + +setup for single sign on with [SSPI](https://wiki.postgresql.org/wiki/Configuring_for_single_sign-on_using_SSPI_on_Windows) + +md5 hash is salted with username in front + +Memory +========================================================= +see whats in the buffer cache with pg_buffercache + +`CREATE EXTENSION pg_buffercache` + +``` +SELECT + c.relname, + COUNT(*) AS buffers +FROM + pg_class c +INNER JOIN pg_buffercache b ON + b.relfilenode = c.relfilenode +INNER JOIN pg_database d ON + ( b.reldatabase = d.oid + AND d.datname = CURRENT_DATABASE()) +GROUP BY + c.relname +ORDER BY + 2 DESC +LIMIT 100; +``` + +Alter Column +========================================================== +ALTER TABLE rlarp.pcore ALTER COLUMN pack SET DATA TYPE numeric USING pack::numeric + +psql binary for latest version is always used but pg_dump is not, you have to set the default version in ~/.postgresqlrc + +PostGIS +========================================================== + +quickstart tutorial + +https://blog.crunchydata.com/blog/postgis-for-newbies + +Move Data Directory +=========================================================== +https://www.digitalocean.com/community/tutorials/how-to-move-a-postgresql-data-directory-to-a-new-location-on-ubuntu-18-04 + +copy the data +`sudo rsync -av /var/lib/postgresql /target_dir` + +rename original as a backup +`sudo mv /var/lib/postgresql/10/main /var/lib/postgresql/10/main.bak` + +point postgres to the new data directory +`sudo vim //etc/postgresql/14/main/postgres.conf` +` data_directory = '/mnt/volume_nyc1_01/postgresql/10/main'` + +remove the old data +`sudo rm -Rf /var/lib/postgresql/10/main.bak` + +Special Aggregates +========================================================== +To extract aggregate definitions can select from `pg_aggregate` + + +SQL for current aggregates I'm using now: +``` +CREATE OR REPLACE FUNCTION public.jsonb_concat( + state jsonb, + concat jsonb) + RETURNS jsonb AS +$BODY$ +BEGIN + --RAISE notice 'state is %', state; + --RAISE notice 'concat is %', concat; + RETURN state || concat; +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; + + +CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr( + state jsonb, + concat jsonb) + RETURNS jsonb AS +$BODY$ +BEGIN + --RAISE notice 'state is %', state; + --RAISE notice 'concat is %', concat; + RETURN SELECT jsonb_agg(state || concat; +END; +$BODY$ + LANGUAGE plpgsql VOLATILE + COST 100; + + +DROP AGGREGATE IF EXISTS public.jsonb_arr_aggc(jsonb); +CREATE AGGREGATE public.jsonb_arr_aggc(jsonb) ( + SFUNC=public.jsonb_concat, + STYPE=jsonb, + INITCOND='[]' +); + +DROP AGGREGATE IF EXISTS public.jsonb_obj_aggc(jsonb); +CREATE AGGREGATE public.jsonb_obj_aggc(jsonb) ( + SFUNC=public.jsonb_concat, + STYPE=jsonb, + INITCOND='{}' +); + +CREATE OR REPLACE FUNCTION public.jsonb_array_add_distinct(_arr jsonb, _add text) RETURNS jsonb AS +$$ +DECLARE + _ret jsonb; + +BEGIN + +SELECT + jsonb_agg(DISTINCT x.ae) +INTO + _ret +FROM + ( + SELECT jsonb_array_elements_text(_arr) ae + UNION ALL + SELECT _add ae + ) x; + +RETURN _ret; + +END; +$$ +language plpgsql +DROP FUNCTION IF EXISTS public.jsonb_array_string_agg; +CREATE FUNCTION public.jsonb_array_string_agg(_arr jsonb, _delim text) RETURNS text AS +$$ +DECLARE + _ret text; + +BEGIN + +SELECT + string_agg(ae.v,_delim) +INTO + _ret +FROM + jsonb_array_elements_text(_arr) ae(v); + +return _ret; + +END; +$$ +LANGUAGE plpgsql; + + +``` + +PSQL +=============================================================== +use -E to show definitions of SQL used for \d commands + +Descriptions +=================================================== +``` +SELECT + c.relname table_name, + td.description table_description, + n.nspname schema_name, + a.attname As column_name, + cd.description column_description +FROM + pg_class As c + INNER JOIN pg_attribute As a ON + c.oid = a.attrelid + LEFT JOIN pg_namespace n ON + n.oid = c.relnamespace + LEFT JOIN pg_tablespace t ON + t.oid = c.reltablespace + LEFT JOIN pg_description As cd ON + cd.objoid = c.oid + AND cd.objsubid = a.attnum + LEFT JOIN pg_description As td ON + td.objoid = c.oid + AND td.objsubid = 0 +WHERE + c.relkind IN('r', 'v') + --AND a.attname = 'd07txn' + AND cd.description like '%Transaction Number%' +ORDER BY + n.nspname, + c.relname, + a.attname +``` + +Foreign Data Wrapper +=============================================================== +``` +CREATE EXTENSION postgres_fdw; + +CREATE SERVER hptrow +FOREIGN DATA WRAPPER postgres_fdw +OPTIONS (host 'hptrow.me', port '54339', dbname 'ubm'); + +CREATE USER MAPPING FOR ptrowbridge +SERVER hptrow +OPTIONS (user 'ptrowbridge', password 'gyaswddh1983'); + +CREATE SCHEMA frlarp; + +IMPORT FOREIGN SCHEMA rlarp +FROM SERVER hptrow INTO frlarp; +``` + + +User DDL +=============================================================== +``` +DROP USER IF EXISTS api; + +SET password_encryption = 'scram-sha-256'; + +CREATE ROLE api WITH + LOGIN + NOSUPERUSER + NOCREATEDB + NOCREATEROLE + INHERIT + NOREPLICATION + CONNECTION LIMIT -1 + PASSWORD 'api'; + +--------------------grant-------------------------------------------------- + +GRANT USAGE ON SCHEMA lgdat TO api; + +GRANT SELECT /*, UPDATE, INSERT, DELETE*/ ON ALL TABLES IN SCHEMA lgdat TO api; + +GRANT USAGE ON ALL SEQUENCES IN SCHEMA lgdat TO api; + +ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT SELECT/*, UPDATE, INSERT, DELETE*/ ON TABLES TO api; + +ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT USAGE ON SEQUENCES TO api; + +---------------------------revoke--------------------------------------- + +REVOKE USAGE ON SCHEMA lgdat FROM api; + +REVOKE USAGE ON SCHEMA lgdat FROM api; + +REVOKE SELECT , UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA lgdat FROM api; + +REVOKE USAGE ON ALL SEQUENCES IN SCHEMA lgdat FROM api; + +ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE SELECT, UPDATE, INSERT, DELETE ON TABLES FROM api; + +ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE USAGE ON SEQUENCES FROM api; +``` \ No newline at end of file diff --git a/postgres/aggregate.md b/postgres/aggregate.md deleted file mode 100644 index bcaddd9..0000000 --- a/postgres/aggregate.md +++ /dev/null @@ -1,49 +0,0 @@ -To extract aggregate definitions can select from `pg_aggregate` - - -SQL for current aggregates I'm using now: -``` -CREATE OR REPLACE FUNCTION public.jsonb_concat( - state jsonb, - concat jsonb) - RETURNS jsonb AS -$BODY$ -BEGIN - --RAISE notice 'state is %', state; - --RAISE notice 'concat is %', concat; - RETURN state || concat; -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; - - -CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr( - state jsonb, - concat jsonb) - RETURNS jsonb AS -$BODY$ -BEGIN - --RAISE notice 'state is %', state; - --RAISE notice 'concat is %', concat; - RETURN SELECT jsonb_agg(state || concat; -END; -$BODY$ - LANGUAGE plpgsql VOLATILE - COST 100; - - -DROP AGGREGATE IF EXISTS public.jsonb_arr_aggc(jsonb); -CREATE AGGREGATE public.jsonb_arr_aggc(jsonb) ( - SFUNC=public.jsonb_concat, - STYPE=jsonb, - INITCOND='[]' -); - -DROP AGGREGATE IF EXISTS public.jsonb_obj_aggc(jsonb); -CREATE AGGREGATE public.jsonb_obj_aggc(jsonb) ( - SFUNC=public.jsonb_concat, - STYPE=jsonb, - INITCOND='{}' -); -``` diff --git a/postgres/concat_distinct_array_values.sql b/postgres/concat_distinct_array_values.sql deleted file mode 100644 index a4ca3b0..0000000 --- a/postgres/concat_distinct_array_values.sql +++ /dev/null @@ -1,27 +0,0 @@ -DO $$ -DECLARE - -_a1 jsonb; -_a2 jsonb; -_a3 jsonb; - -BEGIN - -_a1 := (select '["A","B","A"]'::jsonb); -_a2 := (select '["A","B","A"]'::jsonb); -SELECT - jsonb_agg(DISTINCT x.ae) -INTO - _a3 -FROM - ( - SELECT jsonb_array_elements(_a1) ae - UNION ALL - SELECT jsonb_array_elements(_a2) ae -) x; - -raise notice '%', _a3; - -END; -$$ -language plpgsql diff --git a/postgres/descriptions.sql b/postgres/descriptions.sql deleted file mode 100644 index 7504458..0000000 --- a/postgres/descriptions.sql +++ /dev/null @@ -1,28 +0,0 @@ -SELECT - c.relname table_name, - td.description table_description, - n.nspname schema_name, - a.attname As column_name, - cd.description column_description -FROM - pg_class As c - INNER JOIN pg_attribute As a ON - c.oid = a.attrelid - LEFT JOIN pg_namespace n ON - n.oid = c.relnamespace - LEFT JOIN pg_tablespace t ON - t.oid = c.reltablespace - LEFT JOIN pg_description As cd ON - cd.objoid = c.oid - AND cd.objsubid = a.attnum - LEFT JOIN pg_description As td ON - td.objoid = c.oid - AND td.objsubid = 0 -WHERE - c.relkind IN('r', 'v') - --AND a.attname = 'd07txn' - AND cd.description like '%Transaction Number%' -ORDER BY - n.nspname, - c.relname, - a.attname \ No newline at end of file diff --git a/postgres/foreign_data_wrapper.sql b/postgres/foreign_data_wrapper.sql deleted file mode 100644 index 6468d40..0000000 --- a/postgres/foreign_data_wrapper.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE EXTENSION postgres_fdw; - -CREATE SERVER hptrow -FOREIGN DATA WRAPPER postgres_fdw -OPTIONS (host 'hptrow.me', port '54339', dbname 'ubm'); - -CREATE USER MAPPING FOR ptrowbridge -SERVER hptrow -OPTIONS (user 'ptrowbridge', password 'gyaswddh1983'); - -CREATE SCHEMA frlarp; - -IMPORT FOREIGN SCHEMA rlarp -FROM SERVER hptrow INTO frlarp; diff --git a/postgres/install.sh b/postgres/install.sh deleted file mode 100644 index 9e82c3a..0000000 --- a/postgres/install.sh +++ /dev/null @@ -1,4 +0,0 @@ -sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' -wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - -sudo apt-get update -sudo apt-get -y install postgresql diff --git a/postgres/jsonb_arr_aggcd.sql b/postgres/jsonb_arr_aggcd.sql deleted file mode 100644 index 79bc18a..0000000 --- a/postgres/jsonb_arr_aggcd.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr( - state jsonb, - concat jsonb) - RETURNS jsonb AS -$BODY$ -BEGIN - --RAISE notice 'state is %', state; - --RAISE notice 'concat is %', concat; - RETURN - jsonb_agg(DISTINCT ae) - FROM - ( - SELECT jsonb_array_elements(state) ae - UNION ALL - SELECT jsonb_array_elements(concat) ae - ) x; -END; -$BODY$ -LANGUAGE plpgsql; - -DROP AGGREGATE IF EXISTS public.jsonb_arr_aggcd(jsonb); -CREATE AGGREGATE public.jsonb_arr_aggcd(jsonb) ( - SFUNC=public.jsonb_concat_distinct_arr, - STYPE=jsonb, - INITCOND='[]' -); - diff --git a/postgres/jsonb_array_add_distinct.sql b/postgres/jsonb_array_add_distinct.sql deleted file mode 100644 index 577bdd9..0000000 --- a/postgres/jsonb_array_add_distinct.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE OR REPLACE FUNCTION public.jsonb_array_add_distinct(_arr jsonb, _add text) RETURNS jsonb AS -$$ -DECLARE - _ret jsonb; - -BEGIN - -SELECT - jsonb_agg(DISTINCT x.ae) -INTO - _ret -FROM - ( - SELECT jsonb_array_elements_text(_arr) ae - UNION ALL - SELECT _add ae - ) x; - -RETURN _ret; - -END; -$$ -language plpgsql diff --git a/postgres/jsonb_array_string_agg.sql b/postgres/jsonb_array_string_agg.sql deleted file mode 100644 index 58183e9..0000000 --- a/postgres/jsonb_array_string_agg.sql +++ /dev/null @@ -1,20 +0,0 @@ -DROP FUNCTION IF EXISTS public.jsonb_array_string_agg; -CREATE FUNCTION public.jsonb_array_string_agg(_arr jsonb, _delim text) RETURNS text AS -$$ -DECLARE - _ret text; - -BEGIN - -SELECT - string_agg(ae.v,_delim) -INTO - _ret -FROM - jsonb_array_elements_text(_arr) ae(v); - -return _ret; - -END; -$$ -LANGUAGE plpgsql; diff --git a/postgres/move_data_directory.md b/postgres/move_data_directory.md deleted file mode 100644 index 53e31c4..0000000 --- a/postgres/move_data_directory.md +++ /dev/null @@ -1,14 +0,0 @@ -https://www.digitalocean.com/community/tutorials/how-to-move-a-postgresql-data-directory-to-a-new-location-on-ubuntu-18-04 - -copy the data -`sudo rsync -av /var/lib/postgresql /target_dir` - -rename original as a backup -`sudo mv /var/lib/postgresql/10/main /var/lib/postgresql/10/main.bak` - -point postgres to the new data directory -`sudo vim //etc/postgresql/14/main/postgres.conf` -` data_directory = '/mnt/volume_nyc1_01/postgresql/10/main'` - -remove the old data -`sudo rm -Rf /var/lib/postgresql/10/main.bak` diff --git a/postgres/out_of_memory.md b/postgres/out_of_memory.md deleted file mode 100644 index 76fd8ea..0000000 --- a/postgres/out_of_memory.md +++ /dev/null @@ -1,12 +0,0 @@ -logs from trying to start with not enough memory: -``` -207716 2020-02-11 11:33:02 EST [2969]: [1-1] user=,db=,app=,client=,remote=LOG: starting PostgreSQL 12.1 (Ubuntu 12.1-1.pgdg18.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0, 64-bit -207717 2020-02-11 11:33:02 EST [2969]: [2-1] user=,db=,app=,client=,remote=LOG: listening on IPv4 address "0.0.0.0", port 5030 -207718 2020-02-11 11:33:02 EST [2969]: [3-1] user=,db=,app=,client=,remote=LOG: listening on IPv6 address "::", port 5030 -207719 2020-02-11 11:33:04 EST [2969]: [4-1] user=,db=,app=,client=,remote=LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5030" -207720 2020-02-11 11:33:06 EST [2969]: [5-1] user=,db=,app=,client=,remote=FATAL: could not map anonymous shared memory: Cannot allocate memory -207721 2020-02-11 11:33:06 EST [2969]: [6-1] user=,db=,app=,client=,remote=HINT: This error usually means that PostgreSQL's request for a shared memory segment exceeded available memory, swap space, or huge pages. To reduce the request size (currently 4423614464 bytes ), reduce PostgreSQL's shared memory usage, perhaps by reducing shared_buffers or max_connections. -207722 2020-02-11 11:33:06 EST [2969]: [7-1] user=,db=,app=,client=,remote=LOG: database system is shut down -207723 pg_ctl: could not start server -207724 Examine the log output. -``` \ No newline at end of file diff --git a/postgres/postgis.md b/postgres/postgis.md deleted file mode 100644 index 31b5e56..0000000 --- a/postgres/postgis.md +++ /dev/null @@ -1,3 +0,0 @@ -quickstart tutorial - -https://blog.crunchydata.com/blog/postgis-for-newbies \ No newline at end of file diff --git a/postgres/postgres.md b/postgres/postgres.md deleted file mode 100644 index f24350b..0000000 --- a/postgres/postgres.md +++ /dev/null @@ -1,34 +0,0 @@ -setup for single sign on with [SSPI](https://wiki.postgresql.org/wiki/Configuring_for_single_sign-on_using_SSPI_on_Windows) - -md5 hash is salted with username in front - - -Memory -========================================================= -see whats in the buffer cache with pg_buffercache - -`CREATE EXTENSION pg_buffercache` - -``` -SELECT - c.relname, - COUNT(*) AS buffers -FROM - pg_class c -INNER JOIN pg_buffercache b ON - b.relfilenode = c.relfilenode -INNER JOIN pg_database d ON - ( b.reldatabase = d.oid - AND d.datname = CURRENT_DATABASE()) -GROUP BY - c.relname -ORDER BY - 2 DESC -LIMIT 100; -``` - -Alter Column -========================================================== -ALTER TABLE rlarp.pcore ALTER COLUMN pack SET DATA TYPE numeric USING pack::numeric - -psql binary for latest version is always used but pg_dump is not, you have to set the default version in ~/.postgresqlrc diff --git a/postgres/psql.md b/postgres/psql.md deleted file mode 100644 index 568ac48..0000000 --- a/postgres/psql.md +++ /dev/null @@ -1 +0,0 @@ -use -E to show definitions of SQL used for \d commands \ No newline at end of file diff --git a/postgres/user.sql b/postgres/user.sql deleted file mode 100644 index 18b58f1..0000000 --- a/postgres/user.sql +++ /dev/null @@ -1,39 +0,0 @@ -DROP USER IF EXISTS api; - -SET password_encryption = 'scram-sha-256'; - -CREATE ROLE api WITH - LOGIN - NOSUPERUSER - NOCREATEDB - NOCREATEROLE - INHERIT - NOREPLICATION - CONNECTION LIMIT -1 - PASSWORD 'api'; - ---------------------grant-------------------------------------------------- - -GRANT USAGE ON SCHEMA lgdat TO api; - -GRANT SELECT /*, UPDATE, INSERT, DELETE*/ ON ALL TABLES IN SCHEMA lgdat TO api; - -GRANT USAGE ON ALL SEQUENCES IN SCHEMA lgdat TO api; - -ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT SELECT/*, UPDATE, INSERT, DELETE*/ ON TABLES TO api; - -ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT USAGE ON SEQUENCES TO api; - ----------------------------revoke--------------------------------------- - -REVOKE USAGE ON SCHEMA lgdat FROM api; - -REVOKE USAGE ON SCHEMA lgdat FROM api; - -REVOKE SELECT , UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA lgdat FROM api; - -REVOKE USAGE ON ALL SEQUENCES IN SCHEMA lgdat FROM api; - -ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE SELECT, UPDATE, INSERT, DELETE ON TABLES FROM api; - -ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE USAGE ON SEQUENCES FROM api;