pt/notes
1
0
Fork 0
Code Issues 1 Pull Requests Projects 1 Releases Wiki Activity

vault backup: 2023-01-12 18:01:26

Browse Source
This commit is contained in:
Paul Trowbridge 2023-01-12 18:01:26 -05:00
parent f163583f88
commit be034057e0
15 changed files with 280 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

280
postgres.md Normal file
View File

@ -0,0 +1,280 @@
Install
=========================================================
[PostgreSQL: Linux downloads (Ubuntu)](https://www.postgresql.org/download/linux/ubuntu/)
```
# Create the file repository configuration:
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
# Import the repository signing key:
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
# Update the package lists:
sudo apt-get update
# Install the latest version of PostgreSQL.
# If you want a specific version, use 'postgresql-12' or similar instead of 'postgresql':
sudo apt-get -y install postgresql
```
SSPI
========================================================
setup for single sign on with [SSPI](https://wiki.postgresql.org/wiki/Configuring_for_single_sign-on_using_SSPI_on_Windows)
md5 hash is salted with username in front
Memory
=========================================================
see whats in the buffer cache with pg_buffercache
`CREATE EXTENSION pg_buffercache`
```
SELECT
c.relname,
COUNT(*) AS buffers
FROM
pg_class c
INNER JOIN pg_buffercache b ON
b.relfilenode = c.relfilenode
INNER JOIN pg_database d ON
( b.reldatabase = d.oid
AND d.datname = CURRENT_DATABASE())
GROUP BY
c.relname
ORDER BY
2 DESC
LIMIT 100;
```
Alter Column
==========================================================
ALTER TABLE rlarp.pcore ALTER COLUMN pack SET DATA TYPE numeric USING pack::numeric
psql binary for latest version is always used but pg_dump is not, you have to set the default version in ~/.postgresqlrc
PostGIS
==========================================================
quickstart tutorial
https://blog.crunchydata.com/blog/postgis-for-newbies
Move Data Directory
===========================================================
https://www.digitalocean.com/community/tutorials/how-to-move-a-postgresql-data-directory-to-a-new-location-on-ubuntu-18-04
copy the data
`sudo rsync -av /var/lib/postgresql /target_dir`
rename original as a backup
`sudo mv /var/lib/postgresql/10/main /var/lib/postgresql/10/main.bak`
point postgres to the new data directory
`sudo vim //etc/postgresql/14/main/postgres.conf`
` data_directory = '/mnt/volume_nyc1_01/postgresql/10/main'`
remove the old data
`sudo rm -Rf /var/lib/postgresql/10/main.bak`
Special Aggregates
==========================================================
To extract aggregate definitions can select from `pg_aggregate`
SQL for current aggregates I'm using now:
```
CREATE OR REPLACE FUNCTION public.jsonb_concat(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN SELECT jsonb_agg(state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
DROP AGGREGATE IF EXISTS public.jsonb_arr_aggc(jsonb);
CREATE AGGREGATE public.jsonb_arr_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='[]'
);
DROP AGGREGATE IF EXISTS public.jsonb_obj_aggc(jsonb);
CREATE AGGREGATE public.jsonb_obj_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='{}'
);
CREATE OR REPLACE FUNCTION public.jsonb_array_add_distinct(_arr jsonb, _add text) RETURNS jsonb AS
$$
DECLARE
_ret jsonb;
BEGIN
SELECT
jsonb_agg(DISTINCT x.ae)
INTO
_ret
FROM
(
SELECT jsonb_array_elements_text(_arr) ae
UNION ALL
SELECT _add ae
) x;
RETURN _ret;
END;
$$
language plpgsql
DROP FUNCTION IF EXISTS public.jsonb_array_string_agg;
CREATE FUNCTION public.jsonb_array_string_agg(_arr jsonb, _delim text) RETURNS text AS
$$
DECLARE
_ret text;
BEGIN
SELECT
string_agg(ae.v,_delim)
INTO
_ret
FROM
jsonb_array_elements_text(_arr) ae(v);
return _ret;
END;
$$
LANGUAGE plpgsql;
```
PSQL
===============================================================
use -E to show definitions of SQL used for \d commands
Descriptions
===================================================
```
SELECT
c.relname table_name,
td.description table_description,
n.nspname schema_name,
a.attname As column_name,
cd.description column_description
FROM
pg_class As c
INNER JOIN pg_attribute As a ON
c.oid = a.attrelid
LEFT JOIN pg_namespace n ON
n.oid = c.relnamespace
LEFT JOIN pg_tablespace t ON
t.oid = c.reltablespace
LEFT JOIN pg_description As cd ON
cd.objoid = c.oid
AND cd.objsubid = a.attnum
LEFT JOIN pg_description As td ON
td.objoid = c.oid
AND td.objsubid = 0
WHERE
c.relkind IN('r', 'v')
--AND a.attname = 'd07txn'
AND cd.description like '%Transaction Number%'
ORDER BY
n.nspname,
c.relname,
a.attname
```
Foreign Data Wrapper
===============================================================
```
CREATE EXTENSION postgres_fdw;
CREATE SERVER hptrow
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'hptrow.me', port '54339', dbname 'ubm');
CREATE USER MAPPING FOR ptrowbridge
SERVER hptrow
OPTIONS (user 'ptrowbridge', password 'gyaswddh1983');
CREATE SCHEMA frlarp;
IMPORT FOREIGN SCHEMA rlarp
FROM SERVER hptrow INTO frlarp;
```
User DDL
===============================================================
```
DROP USER IF EXISTS api;
SET password_encryption = 'scram-sha-256';
CREATE ROLE api WITH
LOGIN
NOSUPERUSER
NOCREATEDB
NOCREATEROLE
INHERIT
NOREPLICATION
CONNECTION LIMIT -1
PASSWORD 'api';
--------------------grant--------------------------------------------------
GRANT USAGE ON SCHEMA lgdat TO api;
GRANT SELECT /*, UPDATE, INSERT, DELETE*/ ON ALL TABLES IN SCHEMA lgdat TO api;
GRANT USAGE ON ALL SEQUENCES IN SCHEMA lgdat TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT SELECT/*, UPDATE, INSERT, DELETE*/ ON TABLES TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT USAGE ON SEQUENCES TO api;
---------------------------revoke---------------------------------------
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE SELECT , UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA lgdat FROM api;
REVOKE USAGE ON ALL SEQUENCES IN SCHEMA lgdat FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE SELECT, UPDATE, INSERT, DELETE ON TABLES FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE USAGE ON SEQUENCES FROM api;
```

49
postgres/aggregate.md
View File

@ -1,49 +0,0 @@
To extract aggregate definitions can select from `pg_aggregate`
SQL for current aggregates I'm using now:
```
CREATE OR REPLACE FUNCTION public.jsonb_concat(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN SELECT jsonb_agg(state || concat;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
DROP AGGREGATE IF EXISTS public.jsonb_arr_aggc(jsonb);
CREATE AGGREGATE public.jsonb_arr_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='[]'
);
DROP AGGREGATE IF EXISTS public.jsonb_obj_aggc(jsonb);
CREATE AGGREGATE public.jsonb_obj_aggc(jsonb) (
SFUNC=public.jsonb_concat,
STYPE=jsonb,
INITCOND='{}'
);
```

27
postgres/concat_distinct_array_values.sql
View File

@ -1,27 +0,0 @@
DO $$
DECLARE
_a1 jsonb;
_a2 jsonb;
_a3 jsonb;
BEGIN
_a1 := (select '["A","B","A"]'::jsonb);
_a2 := (select '["A","B","A"]'::jsonb);
SELECT
jsonb_agg(DISTINCT x.ae)
INTO
_a3
FROM
(
SELECT jsonb_array_elements(_a1) ae
UNION ALL
SELECT jsonb_array_elements(_a2) ae
) x;
raise notice '%', _a3;
END;
$$
language plpgsql

28
postgres/descriptions.sql
View File

@ -1,28 +0,0 @@
SELECT
c.relname table_name,
td.description table_description,
n.nspname schema_name,
a.attname As column_name,
cd.description column_description
FROM
pg_class As c
INNER JOIN pg_attribute As a ON
c.oid = a.attrelid
LEFT JOIN pg_namespace n ON
n.oid = c.relnamespace
LEFT JOIN pg_tablespace t ON
t.oid = c.reltablespace
LEFT JOIN pg_description As cd ON
cd.objoid = c.oid
AND cd.objsubid = a.attnum
LEFT JOIN pg_description As td ON
td.objoid = c.oid
AND td.objsubid = 0
WHERE
c.relkind IN('r', 'v')
--AND a.attname = 'd07txn'
AND cd.description like '%Transaction Number%'
ORDER BY
n.nspname,
c.relname,
a.attname

14
postgres/foreign_data_wrapper.sql
View File

@ -1,14 +0,0 @@
CREATE EXTENSION postgres_fdw;
CREATE SERVER hptrow
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (host 'hptrow.me', port '54339', dbname 'ubm');
CREATE USER MAPPING FOR ptrowbridge
SERVER hptrow
OPTIONS (user 'ptrowbridge', password 'gyaswddh1983');
CREATE SCHEMA frlarp;
IMPORT FOREIGN SCHEMA rlarp
FROM SERVER hptrow INTO frlarp;

4
postgres/install.sh
View File

@ -1,4 +0,0 @@
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get -y install postgresql

27
postgres/jsonb_arr_aggcd.sql
View File

@ -1,27 +0,0 @@
CREATE OR REPLACE FUNCTION public.jsonb_concat_distinct_arr(
state jsonb,
concat jsonb)
RETURNS jsonb AS
$BODY$
BEGIN
--RAISE notice 'state is %', state;
--RAISE notice 'concat is %', concat;
RETURN
jsonb_agg(DISTINCT ae)
FROM
(
SELECT jsonb_array_elements(state) ae
UNION ALL
SELECT jsonb_array_elements(concat) ae
) x;
END;
$BODY$
LANGUAGE plpgsql;
DROP AGGREGATE IF EXISTS public.jsonb_arr_aggcd(jsonb);
CREATE AGGREGATE public.jsonb_arr_aggcd(jsonb) (
SFUNC=public.jsonb_concat_distinct_arr,
STYPE=jsonb,
INITCOND='[]'
);

23
postgres/jsonb_array_add_distinct.sql
View File

@ -1,23 +0,0 @@
CREATE OR REPLACE FUNCTION public.jsonb_array_add_distinct(_arr jsonb, _add text) RETURNS jsonb AS
$$
DECLARE
_ret jsonb;
BEGIN
SELECT
jsonb_agg(DISTINCT x.ae)
INTO
_ret
FROM
(
SELECT jsonb_array_elements_text(_arr) ae
UNION ALL
SELECT _add ae
) x;
RETURN _ret;
END;
$$
language plpgsql

20
postgres/jsonb_array_string_agg.sql
View File

@ -1,20 +0,0 @@
DROP FUNCTION IF EXISTS public.jsonb_array_string_agg;
CREATE FUNCTION public.jsonb_array_string_agg(_arr jsonb, _delim text) RETURNS text AS
$$
DECLARE
_ret text;
BEGIN
SELECT
string_agg(ae.v,_delim)
INTO
_ret
FROM
jsonb_array_elements_text(_arr) ae(v);
return _ret;
END;
$$
LANGUAGE plpgsql;

14
postgres/move_data_directory.md
View File

@ -1,14 +0,0 @@
https://www.digitalocean.com/community/tutorials/how-to-move-a-postgresql-data-directory-to-a-new-location-on-ubuntu-18-04
copy the data
`sudo rsync -av /var/lib/postgresql /target_dir`
rename original as a backup
`sudo mv /var/lib/postgresql/10/main /var/lib/postgresql/10/main.bak`
point postgres to the new data directory
`sudo vim //etc/postgresql/14/main/postgres.conf`
` data_directory = '/mnt/volume_nyc1_01/postgresql/10/main'`
remove the old data
`sudo rm -Rf /var/lib/postgresql/10/main.bak`

12
postgres/out_of_memory.md
View File

@ -1,12 +0,0 @@
logs from trying to start with not enough memory:
```
207716 2020-02-11 11:33:02 EST [2969]: [1-1] user=,db=,app=,client=,remote=LOG: starting PostgreSQL 12.1 (Ubuntu 12.1-1.pgdg18.04+1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0, 64-bit
207717 2020-02-11 11:33:02 EST [2969]: [2-1] user=,db=,app=,client=,remote=LOG: listening on IPv4 address "0.0.0.0", port 5030
207718 2020-02-11 11:33:02 EST [2969]: [3-1] user=,db=,app=,client=,remote=LOG: listening on IPv6 address "::", port 5030
207719 2020-02-11 11:33:04 EST [2969]: [4-1] user=,db=,app=,client=,remote=LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5030"
207720 2020-02-11 11:33:06 EST [2969]: [5-1] user=,db=,app=,client=,remote=FATAL: could not map anonymous shared memory: Cannot allocate memory
207721 2020-02-11 11:33:06 EST [2969]: [6-1] user=,db=,app=,client=,remote=HINT: This error usually means that PostgreSQL's request for a shared memory segment exceeded available memory, swap space, or huge pages. To reduce the request size (currently 4423614464 bytes ), reduce PostgreSQL's shared memory usage, perhaps by reducing shared_buffers or max_connections.
207722 2020-02-11 11:33:06 EST [2969]: [7-1] user=,db=,app=,client=,remote=LOG: database system is shut down
207723 pg_ctl: could not start server
207724 Examine the log output.
```

3
postgres/postgis.md
View File

@ -1,3 +0,0 @@
quickstart tutorial
https://blog.crunchydata.com/blog/postgis-for-newbies

34
postgres/postgres.md
View File

@ -1,34 +0,0 @@
setup for single sign on with [SSPI](https://wiki.postgresql.org/wiki/Configuring_for_single_sign-on_using_SSPI_on_Windows)
md5 hash is salted with username in front
Memory
=========================================================
see whats in the buffer cache with pg_buffercache
`CREATE EXTENSION pg_buffercache`
```
SELECT
c.relname,
COUNT(*) AS buffers
FROM
pg_class c
INNER JOIN pg_buffercache b ON
b.relfilenode = c.relfilenode
INNER JOIN pg_database d ON
( b.reldatabase = d.oid
AND d.datname = CURRENT_DATABASE())
GROUP BY
c.relname
ORDER BY
2 DESC
LIMIT 100;
```
Alter Column
==========================================================
ALTER TABLE rlarp.pcore ALTER COLUMN pack SET DATA TYPE numeric USING pack::numeric
psql binary for latest version is always used but pg_dump is not, you have to set the default version in ~/.postgresqlrc

1
postgres/psql.md
View File

@ -1 +0,0 @@
use -E to show definitions of SQL used for \d commands

39
postgres/user.sql
View File

@ -1,39 +0,0 @@
DROP USER IF EXISTS api;
SET password_encryption = 'scram-sha-256';
CREATE ROLE api WITH
LOGIN
NOSUPERUSER
NOCREATEDB
NOCREATEROLE
INHERIT
NOREPLICATION
CONNECTION LIMIT -1
PASSWORD 'api';
--------------------grant--------------------------------------------------
GRANT USAGE ON SCHEMA lgdat TO api;
GRANT SELECT /*, UPDATE, INSERT, DELETE*/ ON ALL TABLES IN SCHEMA lgdat TO api;
GRANT USAGE ON ALL SEQUENCES IN SCHEMA lgdat TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT SELECT/*, UPDATE, INSERT, DELETE*/ ON TABLES TO api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat GRANT USAGE ON SEQUENCES TO api;
---------------------------revoke---------------------------------------
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE USAGE ON SCHEMA lgdat FROM api;
REVOKE SELECT , UPDATE, INSERT, DELETE ON ALL TABLES IN SCHEMA lgdat FROM api;
REVOKE USAGE ON ALL SEQUENCES IN SCHEMA lgdat FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE SELECT, UPDATE, INSERT, DELETE ON TABLES FROM api;
ALTER DEFAULT PRIVILEGES IN SCHEMA lgdat REVOKE USAGE ON SEQUENCES FROM api;