vault backup: 2023-01-12 18:01:26

This commit is contained in:
Paul Trowbridge 2023-01-12 18:01:26 -05:00
parent f163583f88
commit be034057e0
15 changed files with 280 additions and 295 deletions

280
postgres.md Normal file
View File

@ -0,0 +1,280 @@
Install
=========================================================
[PostgreSQL: Linux downloads (Ubuntu)](https://www.postgresql.org/download/linux/ubuntu/)
```
# Create the file repository configuration:
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
# Import the repository signing key:
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
# Update the package lists:
sudo apt-get update
# Install the latest version of PostgreSQL.
# If you want a specific version, use 'postgresql-12' or similar instead of 'postgresql':
sudo apt-get -y install postgresql
```
SSPI
========================================================
setup for single sign on with [SSPI](https://wiki.postgresql.org/wiki/Configuring_for_single_sign-on_using_SSPI_on_Windows)
md5 hash is salted with username in front
Memory
=========================================================
see whats in the buffer cache with pg_buffercache
`CREATE EXTENSION pg_buffercache`
```
SELECT
c.relname,
COUNT(*) AS buffers
FROM
pg_class c
INNER JOIN pg_buffercache b ON
b.relfilenode = c.relfilenode
INNER JOIN pg_database d ON
( b.reldatabase = d.oid
AND d.datname = CURRENT_DATABASE())
GROUP BY
c.relname
ORDER BY
2 DESC
LIMIT 100;
```
Alter Column
==========================================================
ALTER TABLE rlarp.pcore ALTER COLUMN pack SET DATA TYPE numeric USING pack::numeric
psql binary for latest version is always used but pg_dump is not, you have to set the default version in ~/.postgresqlrc
PostGIS
==========================================================
quickstart tutorial
https://blog.crunchydata.com/blog/postgis-for-newbies
Move Data Directory
===========================================================
https://www.digitalocean.com/community/tutorials/how-to-move-a-postgresql-data-directory-to-a-new-location-on-ubuntu-18-04
copy the data
`sudo rsync -av /var/lib/postgresql /target_dir`
rename original as a backup
`sudo mv /var/lib/postgresql/10/main /var/lib/postgresql/10/main.bak`
point postgres to the new data directory
`sudo vim //etc/postgresql/14/main/postgres.conf`
` data_directory = '/mnt/volume_nyc1_01/postgresql/10/main'`
remove the old data
`sudo rm -Rf /var/lib/postgresql/10/main.bak`
Special Aggregates
==========================================================
To extract aggregate definitions can select from `pg_aggregate`
SQL for current aggregates I'm using now:
```
CREATE OR REPLACE FUNCTION public.jsonb_concat(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN SELECT jsonb_agg(state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
DROP AGGREGATE IF EXISTS public.jsonb_arr_aggc(jsonb);
CREATE AGGREGATE public.jsonb_arr_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='[]'
);
DROP AGGREGATE IF EXISTS public.jsonb_obj_aggc(jsonb);
CREATE AGGREGATE public.jsonb_obj_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='{}'
);
CREATE OR REPLACE FUNCTION public.jsonb_array_add_distinct(_arr jsonb, _add text) RETURNS jsonb AS
$$
DECLARE
_ret jsonb;
BEGIN
SELECT
jsonb_agg(DISTINCT x.ae)
INTO
_ret
FROM
(
SELECT jsonb_array_elements_text(_arr) ae
UNION ALL
SELECT _add ae
) x;
RETURN _ret;
END;
$$
language plpgsql
DROP FUNCTION IF EXISTS public.jsonb_array_string_agg;
CREATE FUNCTION public.jsonb_array_string_agg(_arr jsonb, _delim text) RETURNS text AS
$$
DECLARE
_ret text;
BEGIN
SELECT
string_agg(ae.v,_delim)
INTO
_ret
FROM
jsonb_array_elements_text(_arr) ae(v);
return _ret;
END;
$$
LANGUAGE plpgsql;
```
PSQL
===============================================================
use -E to show definitions of SQL used for \d commands
Descriptions
===================================================
```
SELECT
c.relname table_name,
td.description table_description,
n.nspname schema_name,
a.attname As column_name,
cd.description column_description
FROM
pg_class As c
INNER JOIN pg_attribute As a ON
c.oid = a.attrelid
LEFT JOIN pg_namespace n ON
n.oid = c.relnamespace
LEFT JOIN pg_tablespace t ON
t.oid = c.reltablespace
LEFT JOIN pg_description As cd ON
cd.objoid = c.oid
AND cd.objsubid = a.attnum
LEFT JOIN pg_description As td ON
td.objoid = c.oid
AND td.objsubid = 0
WHERE
c.relkind IN('r', 'v')
--AND a.attname = 'd07txn'
AND cd.description like '%Transaction Number%'
ORDER BY
n.nspname,
c.relname,
a.attname
```
Foreign Data Wrapper
===============================================================
```
CREATE EXTENSION postgres_fdw;
CREATE SERVER hptrow
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'hptrow.me', port '54339', dbname 'ubm');
CREATE USER MAPPING FOR ptrowbridge
SERVER hptrow
OPTIONS (user 'ptrowbridge', password 'gyaswddh1983');
CREATE SCHEMA frlarp;
IMPORT FOREIGN SCHEMA rlarp
FROM SERVER hptrow INTO frlarp;
```
User DDL
===============================================================
```
DROP USER IF EXISTS api;
SET password_encryption = 'scram-sha-256';
CREATE ROLE api WITH
LOGIN
NOSUPERUSER
NOCREATEDB
NOCREATEROLE
INHERIT
NOREPLICATION
CONNECTION LIMIT -1
PASSWORD 'api';
--------------------grant--------------------------------------------------
GRANT USAGE ON SCHEMA lgdat TO api;
GRANT SELECT /*, UPDATE, INSERT, DELETE*/ ON ALL TABLES IN SCHEMA lgdat TO api;
GRANT USAGE ON ALL SEQUENCES IN SCHEMA lgdat TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT SELECT/*, UPDATE, INSERT, DELETE*/ ON TABLES TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT USAGE ON SEQUENCES TO api;
---------------------------revoke---------------------------------------
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE SELECT , UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA lgdat FROM api;
REVOKE USAGE ON ALL SEQUENCES IN SCHEMA lgdat FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE SELECT, UPDATE, INSERT, DELETE ON TABLES FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE USAGE ON SEQUENCES FROM api;
```

View File

@ -1,49 +0,0 @@
To extract aggregate definitions can select from `pg_aggregate`
SQL for current aggregates I'm using now:
```
CREATE OR REPLACE FUNCTION public.jsonb_concat(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN SELECT jsonb_agg(state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
DROP AGGREGATE IF EXISTS public.jsonb_arr_aggc(jsonb);
CREATE AGGREGATE public.jsonb_arr_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='[]'
);
DROP AGGREGATE IF EXISTS public.jsonb_obj_aggc(jsonb);
CREATE AGGREGATE public.jsonb_obj_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='{}'
);
```

View File

@ -1,27 +0,0 @@
DO $$
DECLARE
_a1 jsonb;
_a2 jsonb;
_a3 jsonb;
BEGIN
_a1 := (select '["A","B","A"]'::jsonb);
_a2 := (select '["A","B","A"]'::jsonb);
SELECT
jsonb_agg(DISTINCT x.ae)
INTO
_a3
FROM
(
SELECT jsonb_array_elements(_a1) ae
UNION ALL
SELECT jsonb_array_elements(_a2) ae
) x;
raise notice '%', _a3;
END;
$$
language plpgsql

View File

@ -1,28 +0,0 @@
SELECT
c.relname table_name,
td.description table_description,
n.nspname schema_name,
a.attname As column_name,
cd.description column_description
FROM
pg_class As c
INNER JOIN pg_attribute As a ON
c.oid = a.attrelid
LEFT JOIN pg_namespace n ON
n.oid = c.relnamespace
LEFT JOIN pg_tablespace t ON
t.oid = c.reltablespace
LEFT JOIN pg_description As cd ON
cd.objoid = c.oid
AND cd.objsubid = a.attnum
LEFT JOIN pg_description As td ON
td.objoid = c.oid
AND td.objsubid = 0
WHERE
c.relkind IN('r', 'v')
--AND a.attname = 'd07txn'
AND cd.description like '%Transaction Number%'
ORDER BY
n.nspname,
c.relname,
a.attname

View File

@ -1,14 +0,0 @@
CREATE EXTENSION postgres_fdw;
CREATE SERVER hptrow
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'hptrow.me', port '54339', dbname 'ubm');
CREATE USER MAPPING FOR ptrowbridge
SERVER hptrow
OPTIONS (user 'ptrowbridge', password 'gyaswddh1983');
CREATE SCHEMA frlarp;
IMPORT FOREIGN SCHEMA rlarp
FROM SERVER hptrow INTO frlarp;

View File

@ -1,4 +0,0 @@
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get -y install postgresql

View File

@ -1,27 +0,0 @@
CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN
jsonb_agg(DISTINCT ae)
FROM
(
SELECT jsonb_array_elements(state) ae
UNION ALL
SELECT jsonb_array_elements(concat) ae
) x;
END;
$BODY$
LANGUAGE plpgsql;
DROP AGGREGATE IF EXISTS public.jsonb_arr_aggcd(jsonb);
CREATE AGGREGATE public.jsonb_arr_aggcd(jsonb) (
SFUNC=public.jsonb_concat_distinct_arr,
STYPE=jsonb,
INITCOND='[]'
);

View File

@ -1,23 +0,0 @@
CREATE OR REPLACE FUNCTION public.jsonb_array_add_distinct(_arr jsonb, _add text) RETURNS jsonb AS
$$
DECLARE
_ret jsonb;
BEGIN
SELECT
jsonb_agg(DISTINCT x.ae)
INTO
_ret
FROM
(
SELECT jsonb_array_elements_text(_arr) ae
UNION ALL
SELECT _add ae
) x;
RETURN _ret;
END;
$$
language plpgsql

View File

@ -1,20 +0,0 @@
DROP FUNCTION IF EXISTS public.jsonb_array_string_agg;
CREATE FUNCTION public.jsonb_array_string_agg(_arr jsonb, _delim text) RETURNS text AS
$$
DECLARE
_ret text;
BEGIN
SELECT
string_agg(ae.v,_delim)
INTO
_ret
FROM
jsonb_array_elements_text(_arr) ae(v);
return _ret;
END;
$$
LANGUAGE plpgsql;

View File

@ -1,14 +0,0 @@
https://www.digitalocean.com/community/tutorials/how-to-move-a-postgresql-data-directory-to-a-new-location-on-ubuntu-18-04
copy the data
`sudo rsync -av /var/lib/postgresql /target_dir`
rename original as a backup
`sudo mv /var/lib/postgresql/10/main /var/lib/postgresql/10/main.bak`
point postgres to the new data directory
`sudo vim //etc/postgresql/14/main/postgres.conf`
` data_directory = '/mnt/volume_nyc1_01/postgresql/10/main'`
remove the old data
`sudo rm -Rf /var/lib/postgresql/10/main.bak`

View File

@ -1,12 +0,0 @@
logs from trying to start with not enough memory:
```
207716 2020-02-11 11:33:02 EST [2969]: [1-1] user=,db=,app=,client=,remote=LOG: starting PostgreSQL 12.1 (Ubuntu 12.1-1.pgdg18.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0, 64-bit
207717 2020-02-11 11:33:02 EST [2969]: [2-1] user=,db=,app=,client=,remote=LOG: listening on IPv4 address "0.0.0.0", port 5030
207718 2020-02-11 11:33:02 EST [2969]: [3-1] user=,db=,app=,client=,remote=LOG: listening on IPv6 address "::", port 5030
207719 2020-02-11 11:33:04 EST [2969]: [4-1] user=,db=,app=,client=,remote=LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5030"
207720 2020-02-11 11:33:06 EST [2969]: [5-1] user=,db=,app=,client=,remote=FATAL: could not map anonymous shared memory: Cannot allocate memory
207721 2020-02-11 11:33:06 EST [2969]: [6-1] user=,db=,app=,client=,remote=HINT: This error usually means that PostgreSQL's request for a shared memory segment exceeded available memory, swap space, or huge pages. To reduce the request size (currently 4423614464 bytes ), reduce PostgreSQL's shared memory usage, perhaps by reducing shared_buffers or max_connections.
207722 2020-02-11 11:33:06 EST [2969]: [7-1] user=,db=,app=,client=,remote=LOG: database system is shut down
207723 pg_ctl: could not start server
207724 Examine the log output.
```

View File

@ -1,3 +0,0 @@
quickstart tutorial
https://blog.crunchydata.com/blog/postgis-for-newbies

View File

@ -1,34 +0,0 @@
setup for single sign on with [SSPI](https://wiki.postgresql.org/wiki/Configuring_for_single_sign-on_using_SSPI_on_Windows)
md5 hash is salted with username in front
Memory
=========================================================
see whats in the buffer cache with pg_buffercache
`CREATE EXTENSION pg_buffercache`
```
SELECT
c.relname,
COUNT(*) AS buffers
FROM
pg_class c
INNER JOIN pg_buffercache b ON
b.relfilenode = c.relfilenode
INNER JOIN pg_database d ON
( b.reldatabase = d.oid
AND d.datname = CURRENT_DATABASE())
GROUP BY
c.relname
ORDER BY
2 DESC
LIMIT 100;
```
Alter Column
==========================================================
ALTER TABLE rlarp.pcore ALTER COLUMN pack SET DATA TYPE numeric USING pack::numeric
psql binary for latest version is always used but pg_dump is not, you have to set the default version in ~/.postgresqlrc

View File

@ -1 +0,0 @@
use -E to show definitions of SQL used for \d commands

View File

@ -1,39 +0,0 @@
DROP USER IF EXISTS api;
SET password_encryption = 'scram-sha-256';
CREATE ROLE api WITH
LOGIN
NOSUPERUSER
NOCREATEDB
NOCREATEROLE
INHERIT
NOREPLICATION
CONNECTION LIMIT -1
PASSWORD 'api';
--------------------grant--------------------------------------------------
GRANT USAGE ON SCHEMA lgdat TO api;
GRANT SELECT /*, UPDATE, INSERT, DELETE*/ ON ALL TABLES IN SCHEMA lgdat TO api;
GRANT USAGE ON ALL SEQUENCES IN SCHEMA lgdat TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT SELECT/*, UPDATE, INSERT, DELETE*/ ON TABLES TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT USAGE ON SEQUENCES TO api;
---------------------------revoke---------------------------------------
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE SELECT , UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA lgdat FROM api;
REVOKE USAGE ON ALL SEQUENCES IN SCHEMA lgdat FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE SELECT, UPDATE, INSERT, DELETE ON TABLES FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE USAGE ON SEQUENCES FROM api;