PostgreSQL 8.4 Update

LOCAL DEVELOPER DAY '09 /Winter ( @ , 2009 ) http://www.local.gr.jp/?php%2Fhistory%2F%C2%E803%B2%F3%28LDD%2709%2FWinter%29 PostgreSQL 8.4 Update PostgreSQL / NTT OSS 2009.2.14

PostgreSQL 8.4 8.0 8.3 8.4 2009 7 Ingress Ingress PostgreSQL PostgreSQL 8.1 6.0 6.0 •• 1977 7.4 7.4 •2 (2PC) 8.3 1986 1996 •2 (2PC) 2000 •• •HOT: •HOT: 2003 2004 • • POSTGRES POSTGRES 2005 •VACUUM •VACUUM // 2006 7.3 7.3 2007 2008 8.0 •• 2009 •• (PITR) (PITR) •• 8.2 •Windows •CPU •CPU •Windows • • • • •GIN: •GIN: 8.4 2

8.4 / HOT XML autovacuum 8.3 2008 WITH SQL pgtune 8.4 Window Visibility Map pg_reorg 2009 / 8.5 2010 ? 3

PostgreSQL 8.4 : / / / WITH Window / SQL pgtune Visibility Map pg_reorg 4

: / CHECK 8.2 • SELECT, UPDATE, DELETE • 1 2 3 4 4× ( ) 8.4 constraint_exclusion = partition CHECK EXECUTE USING in PL/pgSQL INSERT 5

: (EXECUTE USING) / INSERT : tbl CREATE TRIGGER BEFORE INSERT ( ) CHECK CHECK('2009-01-01' <= insert_time 1 2 3 4 AND insert_time < '2009-02-01') : tbl_[ ][ ] CREATE FUNCTION insert_trigger() RETURNS TRIGGER AS $$ DECLARE part text; (8.3 BEGIN IF ) part := 'tbl_' || to_char(new.insert_time, 'YYYYMM'); EXECUTE 'INSERT INTO ' || part || ' VALUES(($1).*)' USING new; RETURN NULL; END; $$ EXECUTE USING LANGUAGE plpgsql; SQL new 6

: / effective_io_concurrency I/O RAID I/O Bitmap Heap Scan ◎1 SQL OLTP × OLAP posix_fadvise(WILLNEED) POSIX ( Linux) / Windows 7

/ pg_restore -m, --multi-thread = number-of-threads PostgreSQL CPU PostgreSQL pg_dump --format=custom D pg_restore R R R R ( ) ( ) 8

PostgreSQL 8.4 : / WITH Window / SQL pgtune Visibility Map pg_reorg 9

WITH ( ) WITH, WITH RECURSIVE ID =# TABLE tree; id | parent 1 ----+-------- 1 | 2 | 1 2 3 self-join 3 | 1 4 | 3 4 ... =# WITH RECURSIVE r AS ( SELECT * FROM tree WHERE id = 1 UNION ALL SELECT tree.* FROM tree, r WHERE tree.parent = r.id ) SELECT * FROM r ORDER BY id; id | parent ----+-------- 1 | 2 | 1 3 | 1 4 | 3 (4 rows) 10

WITH ( ): ML (!?) WITH RECURSIVE Z(IX, IY, CX, CY, X, Y, I) AS ( SELECT IX, IY, X::float, Y::float, X::float, Y::float, 0 FROM (SELECT -2.2 + 0.031 * i, i FROM generate_series(0, 101) AS i) AS xgen(x, ix), (SELECT -1.5 + 0.031 * i, i FROM generate_series(0, 101) AS i) AS ygen(y, iy) UNION ALL SELECT IX, IY, CX, CY, X * X - Y * Y + CX AS X, Y * X * 2 + CY, I + 1 FROM Z WHERE X * X + Y * Y < 16::float AND I < 100 ) SELECT array_to_string(array_agg(substring(' .,,,-----++++%%%%@@@@#### ', least(greatest(I,1),27), 1)),'') FROM ( SELECT IX, IY, max(I) AS I FROM Z GROUP BY IY, IX ORDER BY IY, IX ) AS ZT GROUP BY IY Q. ? ORDER BY IY; 11

WITH ( ): .................................................................................... ....................................................................................... ......................................................................................... ........................................................................................... ....................................................,,,,,,,,,................................. ................................................,,,,,,,,,,,,,,,,,,............................. ..............................................,,,,,,,,,,,,,,,,,,,,,,,,.......................... ............................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,........................ ..........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...................... .........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,.................... ........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,................... .......................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,................. .......................................,,,,,,,,,,,,,,,,,,,,,,,,--,,,,,,,,,,,,,,,,,,,,................ ......................................,,,,,,,,,,,,,,,,,,,,,,,,,,-+--,,,,,,,,,,,,,,,,,,,............... ....................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----,,,,,,,,,,,,,,,,,,,.............. ...................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,--- -----,,,,,,,,,,,,,,,,,............. .................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---++--++,,,,,,,,,,,,,,,,,,............ ................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----%++---,,,,,,,,,,,,,,,,,............ ..............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+%----,,,,,,,,,,,,,,,,,,........... .............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----- %%+----,,,,,,,,,,,,,,,,,,.......... ...........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---%-+% ----,,,,,,,,,,,,,,,,,,,......... ..........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+ +## %+%---,,,,,,,,,,,,,,,,,,......... ........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----# # +---,,,,,,,,,,,,,,,,,,........ .......................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------% %-----,,,,,,,,,,,,,,,,,........ .....................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---------+ ------,,,,,,,,,,,,,,,,,....... ....................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----------+@ +-----------,,,,,,,,,,,,....... ..................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----@-------++ ++-----------,,,,,,,,,,,,...... .................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--+@% ---+ +@%%@ %%+@+@%------+-,,,,,,,,,,,...... ................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---- # ++% % @-----++--,,,,,,,,,,,..... ..............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----+ % %%++ %+%@-,,,,,,,,,,,..... .............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+# #% ++-,,,,,,,,,,,,.... ............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,------+ @---,,,,,,,,,,,,.... ..........,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------++% ---,,,,,,,,,,,,.... .........,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--------+ + %+---,,,,,,,,,,,,,... ........,,,,,,,,,,,,,,,,,,,,,--------------------@ +----,,,,,,,,,,,,... A. .......,,,,,,,,,,,,,,,,,,,,,,- +-----------------+ ----,,,,,,,,,,,,... .......,,,,,,,,,,,,,,,,,,,,,--++------+---------+% +++--,,,,,,,,,,,,.. ......,,,,,,,,,,,,,,,,,,,,,,--%+-----++--------- #+-,,,,,,,,,,,,.. .....,,,,,,,,,,,,,,,,,,,,,,----#%++--+@ -+-----+% --,,,,,,,,,,,,.. .....,,,,,,,,,,,,,,,,,,,,,,-----+## ++@ + +----% +--,,,,,,,,,,,,,.. ....,,,,,,,,,,,,,,,,,,,,,,------+@ @ @@++++# +--,,,,,,,,,,,,,.. ....,,,,,,,,,,,,,,,,,,,,,-------% #++% -,,,,,,,,,,,,,.. ...,,,,,,,,,,,,,,,,,,,,,------++%# %%@ %-,,,,,,,,,,,,,,. ...,,,,,,,,,,,,,,,,,,,--------+ % +--,,,,,,,,,,,,,,. ...,,,,,,,,,,,,,,,,,,-----+--++@ # --,,,,,,,,,,,,,,. ..,,,,,,,,,,,,,,,,,-------%+++% @--,,,,,,,,,,,,,,,. ..,,,,,,,,,,,-------------+ @#@ ---,,,,,,,,,,,,,,,. ..,,,,,,,,,---@--------@-+% +---,,,,,,,,,,,,,,,. ..,,,,,------- +-++++-+%%% +----,,,,,,,,,,,,,,,. ..,,,,,,------%--------++% +----,,,,,,,,,,,,,,,. ..,,,,,,,,,,--+----------++# ---,,,,,,,,,,,,,,,. ..,,,,,,,,,,,,------------+@@@% +--,,,,,,,,,,,,,,,. ..,,,,,,,,,,,,,,,,,------- +++% %--,,,,,,,,,,,,,,,. ...,,,,,,,,,,,,,,,,,,---------+@ @ --,,,,,,,,,,,,,,. ...,,,,,,,,,,,,,,,,,,,,------- # %@ +--,,,,,,,,,,,,,,. ...,,,,,,,,,,,,,,,,,,,,,-------++@ %+ %-,,,,,,,,,,,,,,. ....,,,,,,,,,,,,,,,,,,,,,------- %++% %-,,,,,,,,,,,,,.. ....,,,,,,,,,,,,,,,,,,,,,,------+# %# #@ ++++ +--,,,,,,,,,,,,,.. .....,,,,,,,,,,,,,,,,,,,,,,-----+ %%++% +@+----+ +--,,,,,,,,,,,,,.. .....,,,,,,,,,,,,,,,,,,,,,,,---%+++--+#+--------% #--,,,,,,,,,,,,.. ......,,,,,,,,,,,,,,,,,,,,,,--++-----%%--------- @#--,,,,,,,,,,,,.. .......,,,,,,,,,,,,,,,,,,,,,---------------------+@ +-++,,,,,,,,,,,,... ........,,,,,,,,,,,,,,,,,,,,,--------------------+ ----,,,,,,,,,,,,... .........,,,,,,,,,,,,,,,,,,,,----,,,------------- #+----,,,,,,,,,,,,... ..........,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------+ + +---,,,,,,,,,,,,,... ...........,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--------+%# ............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,------+# .............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+# ..............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+% %+@ ................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----% %@++ @ + #---,,,,,,,,,,,,.... @---,,,,,,,,,,,,.... @--,,,,,,,,,,,,.... %+-+ +++%-,,,,,,,,,,,..... # % -----++-,,,,,,,,,,,,..... WITH SQL .................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-- ++ ---+ + +%@ %++++++------%-,,,,,,,,,,,...... ...................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---- -------++ +------------,,,,,,,,,,,,...... ....................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----------+% +--------,,,,,,,,,,,,,,,....... ......................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--------+# -----,,,,,,,,,,,,,,,,,,....... .......................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------+ #----,,,,,,,,,,,,,,,,,,........ .........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----+% %#---,,,,,,,,,,,,,,,,,,,........ ..........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+%+%@ %+%%--,,,,,,,,,,,,,,,,,,......... ............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+-+% %----,,,,,,,,,,,,,,,,,,.......... .............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+%@+---,,,,,,,,,,,,,,,,,,,.......... ...............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+%----,,,,,,,,,,,,,,,,,,........... ................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----%+ +--,,,,,,,,,,,,,,,,,............ ..................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---++----,,,,,,,,,,,,,,,,,............. ...................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,---@-----,,,,,,,,,,,,,,,,,............. .....................................,,,,,,,,,,,,,,,,,,,,,,,,,,,-----,,,,,,,,,,,,,,,,,,,.............. .....................................,,,,,,,,,,,,,,,,,,,,,,,,,,--%,,,,,,,,,,,,,,,,,,,,............... .......................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,................. ........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,.................. ........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,................... .........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,.................... ..........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...................... ............................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,........................ .............................................,,,,,,,,,,,,,,,,,,,,,,,,.......................... ................................................,,,,,,,,,,,,,,,,,............................. .....................................................,,,,.................................... ........................................................................................... ......................................................................................... ...................................................................................... .................................................................................... ................................................................................. .............................................................................. ........................................................................... ........................................................................ 12

WITH (vs. CONNECT BY) Oracle CONNECT BY / contrib/tablefunc connectby() Oracle : CONNECT BY SELECT level, empno, mgr, ename, job FROM emp START WITH mgr IS NULL CONNECT BY PRIOR empno = mgr PostgreSQL ~8.3 : connectby() SELECT t.level, e.empno, e.mgr, e.ename, e.job FROM emp e, (SELECT * FROM connectby('emp', 'empno', 'mgr', 'mgr', 7839, 0, '~') AS cb(empno int, mgr int, level int, branch text, pos int)) t WHERE e.empno = t.empno; PostgreSQL 8.4 : WITH RECURSIVE SQL WITH RECURSIVE r AS ( SELECT 1 AS level, * FROM emp WHERE mgr IS NULL UNION ALL SELECT r.level + 1, emp.* FROM emp, r WHERE r.empno = emp.mgr ) SELECT * FROM r; 13

Window : GROUP BY (...) OVER (PARTITION BY ...) (...) OVER (ORDER BY ...) row_number() rank() ( ) dense_rank() ( ) percent_rank() (% ) cume_dist() ntile(N) (1..N ) self-join lag(value, ...) lead(value, ...) first_value(value) 14

Window : SELECT row_number() OVER (), * FROM (SELECT * FROM tbl ORDER BY sortkey) AS t; 10% SELECT ... FROM (SELECT *, cume_dist() OVER (ORDER BY i) AS rank FROM tbl) AS t WHERE rank <= 0.1; LIMIT 10% LIMIT count(*) / 10 SELECT id, , , rank() OVER (PARTITION BY ORDER BY ) FROM tbl; 15

WITH + Window (2/4) locktag() CREATE FUNCTION locktag(pg_catalog.pg_locks) RETURNS text AS $$ SELECT $1.locktype || ' ' || CASE $1.locktype WHEN 'relation' THEN $1.database || ' ' || $1.relation WHEN 'extend' THEN $1.database || ' ' || $1.relation WHEN 'page' THEN $1.database || ' ' || $1.relation || ' ' || $1.page WHEN 'tuple' THEN $1.database || ' ' || $1.relation || ' ' || $1.page || ' ' || $1.tuple WHEN 'transactionid' THEN $1.transactionid::text WHEN 'virtualxid' THEN $1.virtualxid -- FIXME: for object, userlock and advisory END $$ LANGUAGE sql IMMUTABLE STRICT; 17

WITH + Window (3/4) pg_lock_chain row_number() CREATE VIEW pg_lock_chain AS WITH RECURSIVE r AS ( SELECT *, locktag(pg_locks), row_number() OVER () AS chain, 1 AS level FROM pg_locks WHERE NOT granted UNION ALL SELECT s.*, locktag(s), r.chain, r.level + 1 FROM r, pg_locks s WHERE (locktag(s) = r.locktag AND s.granted AND NOT r.granted AND s.pid <> r.pid) OR (s.pid = r.pid AND NOT s.granted AND r.granted) ) SELECT * FROM r; locktag 18

WITH + Window (4/4) pg_lock_chain =# SELECT chain, level, locktag, pid, mode, granted FROM pg_lock_chain ORDER BY chain, level; chain | level | locktag | pid | mode | granted -------+-------+-----------------------+------+---------------+--------- 1 | 1 | transactionid 9930 | 3976 | ShareLock | f 1 | 2 | transactionid 9930 | 3912 | ExclusiveLock | t 2 | 1 | tuple 21509 22126 0 1 | 1992 | ExclusiveLock | f 2 | 2 | tuple 21509 22126 0 1 | 3976 | ExclusiveLock | t 2 | 3 | transactionid 9930 | 3976 | ShareLock | f 2 | 4 | transactionid 9930 | 3912 | ExclusiveLock | t (6 rows) chain 1 chain 2 pid=3976 pid=1992 pid=3912 pid=3912 pid=3976 pid=3912 19

/ PostgreSQL 8.2 CREATE CAST (integer AS varchar) WITH INOUT AS IMPLICIT; DBMS AS SELECT [AS] FROM [AS] DISTINCT GROUP BY or PL/pgSQL CASE 20

=# SELECT *, to_tsvector('japanese', t) FROM tbl; id | t | to_tsvector ----+------------+---------------- 1 | | ' ':1 2 | | ' ':1 (2 rows) =# SELECT * FROM tbl WHERE to_tsvector('japanese', t) @@ to_tsquery('japanese', ' '); id | t ----+---------- 2 | (1 row) =# SELECT * FROM tbl WHERE to_tsvector('japanese', t) @@ to_tsquery('japanese', ' :*'); id | t ----+------------ 1 | :* 2 | (2 rows) N-gram ( :Ludia/Senna) 21

SQL : pg_stat_statements : SQL pg_stat_user_functions : SQL pgFouine PHP PostgreSQL ( ) SQL ◎ ☺ (HTML ) × × / DB 30% 23

SQL : postgresql.conf shared_preload_libraries = 'pg_stat_statements' prepared pgbench (Prepared Statement) $ pgbench -c10 -t300 -M prepared query mode: prepared number of transactions actually processed: 3000/3000 SQL =# SELECT query, calls, total_time, rows FROM pg_stat_statements ORDER BY total_time DESC LIMIT 3; query | calls | total_time | rows --------------------------------------------------------------+-------+-------------------+------ UPDATE branches SET bbalance = bbalance + $1 WHERE bid = $2; | 3000 | 35.9654100452473 | 3000 UPDATE tellers SET tbalance = tbalance + $1 WHERE tid = $2; | 3000 | 34.7969816235719 | 3000 UPDATE accounts SET abalance = abalance + $1 WHERE aid = $2; | 3000 | 0.660384690059746 | 3000 (3 rows) SQL ◎ ◎ 3% × HTML 24

Visibility Map : VACUUM PostgreSQL VACUUM ? ID ID 001 A 10 001 A 10 002 B 8 UPDATE 003 C 20 003 C 20 VACUUM 002 B 7 002 B 7 8.4 VACUUM Visibility Map VACUUM + Free Space Map 8.3 HOT ? HOT = Visibility Map = 25

VACUUM Visibility Map (VACUUM ) 8.4 VACUUM VACUUM Free Space Map (VACUUM ) 8.4 DB Visibility Map Free Space Map : (< ID>[.N]) (< ID>_vm) (< ID>_fsm) 1bit/ 0% 50% 1byte/ 25% Visibility Map VACUUM Free Space Map (shared_buffers) I/O 26

SQL SQL id value 1 (2392) 2 (pid=1956) BEGIN; BEGIN; 1 A UPDATE tbl SET value = 'AA' 2 B WHERE id = 1; UPDATE tbl SET value = 'BB' WHERE id = 2; UPDATE tbl SET value = 'BBB' WHERE id = 2; UPDATE tbl SET value = 'AAA' WHERE id = 1; ERROR: deadlock detected DETAIL: Process 1956 waits for ShareLock on transaction 8214; blocked by process 2392. Process 2392 waits for ShareLock on transaction 8215; blocked by process 1956. Process 1956: UPDATE tbl SET value = 'AAA' WHERE id = 1; Process 2392: UPDATE tbl SET value = 'BBB' WHERE id = 2; 27

pgtune : (DW, OLTP, Web, Mixed, Desktop) ( ) postgresql.conf postgresql.conf =Mixed, =3GB pgtune in max_connections 100 80 shared_buffers 32MB 480MB pgtune wal_buffers 64kB 8MB out work_mem 1MB 12MB maintenance_work_mem 16MB 120MB postgresql.conf postgresql.conf checkpoint_segments 3 16 WAL checkpoint_completion_target 0.5 0.9 effective_cache_size 128MB 1408MB constraint_exclusion partition on default_statistics_target 100 50 http://pgfoundry.org/projects/pgtune/ 29

pg_reorg : / DB VACUUM INSERT UPDATE DELETE ( ) ( 2GB) ■ clusterdb 274 ■ pg_reorg 17 PostgreSQL: 8.2, 8.3, 8.4 0 100 200 300 OS: Linux, Windows http://reorg.projects.postgresql.org/index-ja.html 30

pgbench : pgbench TPC-B SQL pgbench (-T) (-t) (-M) simple, extended, prepared 31

PostgreSQL 8.4 ( ) 2009 7 / , , WITH , Window , / SQL , Visibility Map, pgtune, pg_reorg 32

TRUNCATE TRIGGER SQL/MED pg_terminate_backend() contrib/auto_explain autovacuum TOAST VACUUM pgAdmin III pgScript 33

ToDo ( ) http://wiki.postgresql.org/wiki/Todo PostgreSQL 8.4 Development Plan ( ) http://wiki.postgresql.org/wiki/PostgreSQL_8.4_Dev elopment_Plan Waiting for 8.4 ( blog) http://www.depesz.com/ 34

PostgreSQL 8.4 Update

by Takahiro Itagaki on Jun 23, 2010

Accessibility

Categories

Tags

Upload Details

Usage Rights

Statistics

PostgreSQL 8.4 Update — Presentation Transcript