OpenWorld 2018 - Common Application Developer Disasters

Copyright © 2018, Oracle and/or its affiliates. All rights reserved. |
Don't discombobulate your database
Connor McDonald
Developer Advocate

Connor McDonald

Copyright © 2018, Oracle and/or its affiliates. All rights reserved. |
Stuff
youtube bit.ly/youtube-connor
blog bit.ly/blog-connor
twitter bit.ly/twitter-connor
400+ posts mainly on database & development
250 technical videos, new uploads every week
rants and raves on tech and the world :-)

etc...
facebook bit.ly/facebook-connor
linkedin bit.ly/linkedin-connor
instagram bit.ly/instagram-connor
slideshare bit.ly/slideshare-connor

9https://asktom.oracle.com/officehours

100 hours of free access (so far)
10

12
really silly things
people do to their databases

15
1) the importance of failure

16
developer
team lead
DBA
proj mgr
director
CIO
CEO
“hmmm, my code has a fatal catastrophic bug”
“oh my god! All of the data is corrupt”
“we’ve got some data corruption”
“some small issues with data quality”
“moving forward with data challenges”
“maximising opportunities in data space”
“we pride ourselves on perfect data”

23
responsibility lies with us

... all performance problems are our fault

a little bit of history on servers ...

machines are scary fast today
36
perf.sql

we can't blame the hardware anymore
37

43
https://tinyurl.com/be-nicer

46
PROCEDURE string_check (p_text_in varchar2,p_ret out number) is
char_list varchar2(255);
v_single_char varchar2(1);
v_in_string number;
cursor c1(m number) is
select substr(char_list,m,1) from dual;
cursor c2(v varchar2) is
select instr(p_text_in,v) from dual;
BEGIN
p_ret := 0;
for i in 1 .. 45 loop
char_list := char_list || chr(i);
end loop;
char_list := char_list || chr(47);
for i in 58 .. 255 loop
char_list := char_list || chr(i);
end loop;
for i in 1 .. length(char_list) loop
open c1(i);
fetch c1 into v_single_char;
close c1;
open c2(v_single_char);
fetch c2 into v_in_string;
close c2;
if v_in_string > 0 then
p_ret := 1;
end if;
end loop;
END;

47
ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+/!@#$%^&*......’
string_check('178')178
select substr(char_list,1,1) from dual
select instr( ' ', ' ') from dual
char_list = ‘
select substr(char_list,2,1) from dualselect substr(char_list,3,1) from dual

49
SQL> variable x number
SQL> exec string_check('123',:x)
PL/SQL procedure successfully completed.
SQL> print x
X
----------
0
SQL> exec string_check ('12C3',:x)
SQL> print x
X
----------
1
pass a number,
return 0
pass a non-number,
return 1

50
“check if numeric” function

52
External
System #1
External
System #2
External
System #3
numeric data expected
STRING_CHECK
Our
System

53
every execution
244 "select substr from dual"
244 "select instr from dual"

54
2440 logical I/O’s
per execution
SQL> select sql_text, buffer_gets, executions
2 from v$sqlstats
3 where buffer_gets > 1000000
4 order by 2
5 /
SQL_TEXT BUFFER_GETS EXECUTIONS
-------------------------------------------- ------------- -----------
SELECT SUBSTR(:b1,:b2,1) FROM DUAL ######### ########
SELECT INSTR(:b1,:b2) FROM DUAL ######### ########

55
2440 logical I/O’s
per execution
SQL> select sql_text, buffer_gets, executions
2 from v$sqlstats
3 where buffer_gets > 1000000
4 order by 2
5 /
SQL_TEXT BUFFER_GETS EXECUTIONS
-------------------------------------------- ------------- -----------
SELECT SUBSTR(:b1,:b2,1) FROM DUAL ######### ########
SELECT INSTR(:b1,:b2) FROM DUAL ######### ########
BUFFER_GETS EXECUTIONS
------------- -----------
1612842838 322569518
1612851497 322571313

57
oracle has an “is_number”

59
procedure STRING_CHECK(p_text_in varchar2, p_ret out number) is
begin
p_ret := validate_conversion(p_text_in as number);
end;

60
function CLEAN_NUMBER(p_text_in varchar2) return number is
begin
return cast(p_text_in as number default -1 on conversion error );
end;
perf2.sql

62
procedure STRING_CHECK(p_text_in varchar2, p_ret out number) is
x number;
begin
x := to_number(p_text_in);
p_ret := 0;
exception
when others then p_ret := 1;
end;
no logical I/O
more accurate

64
“I didn’t know about TO_NUMBER”

65
importance of load testing
single execution
100 executions

67
I know what you're thinking ...

68
"LOL...
select from dual...
what....a......dufus"

69
“we would never do anything that stupid”

common mistake #1
70
hidden costs

SQL> declare
2 x varchar2(30);
3 begin
4 x := user;
5 end;
6 /

SQL> create table PEOPLE
2 ( person_id number,
3 forename1 varchar2(20),
4 surname varchar2(30),
5 created_by varchar2(10) default USER );
Table created.

75
"what about when a person is modified?"

SQL> alter table PEOPLE add
2 modified_by varchar2(10) default USER;
Table altered.

SQL> create or replace
2 trigger WHO_DID_THIS_ROW
3 before insert or update
4 on PEOPLE
5 for each row
6 begin
7 if inserting then :new.created_by := user; end if;
8 :new.modified_by := user;
9 end;
10 /
Trigger created.

importance of load testing
single execution
100 executions

insert into people
select rownum,'x','y',null,null
from dba_objects
call count cpu elapsed disk query current rows
------- ------ -------- ---------- ---------- ---------- ---------- ----------
Parse 1 0.03 0.02 0 0 0 0
Execute 1 8.53 8.61 0 26870 2685 78703
Fetch 0 0.00 0.00 0 0 0 0
------- ------ -------- ---------- ---------- ---------- ---------- ----------
total 2 8.56 8.63 0 26870 2685 78703

SELECT USER
FROM SYS.DUAL
------- ------ -------- ---------- ---------- ---------- ---------- ----------
Parse 1 0.00 0.00 0 0 0 0
Execute 157406 0.84 0.81 0 0 0 0
Fetch 157406 0.17 0.18 0 0 0 157406
------- ------ -------- ---------- ---------- ---------- ---------- ----------
total 314813 1.01 1.00 0 0 0 157406

SQL> create or replace
2 trigger WHO_DID_THIS_ROW
3 before insert or update
4 on PEOPLE
5 for each row
6 begin
7 if inserting then
8 :new.created_by := sys_context('USERENV','SESSION_USER');
9 end if;
10 :new.modified_by := sys_context('USERENV','SESSION_USER') ;
11 end;
12 /
Trigger created.

insert into people
select rownum,'x','y',null,null
from dba_objects
------- ------ -------- ---------- ---------- ---------- ---------- ----------
Parse 1 0.00 0.02 0 0 0 0
Execute 1 0.67 0.67 0 26864 2646 78703
Fetch 0 0.00 0.00 0 0 0 0
------- ------ -------- ---------- ---------- ---------- ---------- ----------
total 2 0.67 0.69 0 26864 2646 78703

while we're talking triggers …
85

common mistake #2
86
silent data corruption

“Readers don’t block writers"
“Writers don't block readers"

Maximum concurrency…
… very few locking worries

SQL> update EMP
2 set SAL = SAL * 1.10
3 /
14 rows updated.
SQL> rollback;
Rollback complete.
how ?
93

SQL> update EMP
2 set SAL = SAL * 1.10
3 /
undo header
smith $1,0001
block 2754
jones $9002
brown $1,5003
wells $2,0004
wilson $1,0005
block 2754, row 1,
sal $1000
block 2754, row 2,
sal $900
block 2754, row 3,
sal $1500
block 2754, row 4,
sal $2000
uba 5
5
$1,100
$990
$1,650
$2,200
uba 5.1uba 5.2
95

SQL> rollback;
undo header
smith $1,0001
block 2754
jones $9002
brown $1,5003
wells $2,0004
wilson $1,0005
block 2754, row 1,
sal $1000
block 2754, row 2,
sal $900
block 2754, row 3,
sal $1500
block 2754, row 4,
sal $2000
uba 5
5
$1,100
$990
$1,650
$2,200
uba 5.1uba 5.2

we can use this stuff for queries!
(Oracle version 4)
98

select * from emp
where hiredate > '01/01/2004'
update emp set …
where empno = 1234;
Block 3217
"I need block 3217…
… as it was at 9:00"
9:00
9:03
9:05
session 1 session 2
...and time = scn

System Change Number
(the block “timestamp”)

Session 1
Request
Block 3217,
SCN 4567192
Block 3217,
SCN 4567234
"ABC"
Block 3217,
SCN 4567003,
change ABC => XYZ
undo segment block(s)
No good..too new
take a copy of the block
Locate
apply undo
Block 3217,
SCN 4567234
"ABC"
Block 3217,
SCN 4567234
"ABC"
Block 3217,
SCN 4567003
"XYZ"
Block 3217,
SCN 4567003
"XYZ"
Session 2
update emp
set ename = "ABC"
where empno = 1234;
commit;
Done !
104

“I’m a developer…
…who cares about all that?”

data integrity
... true story
109

110
Manufacturer
Product
alter table add primary key
alter table add foreign key
“ensure that every
product corresponds to
a valid manufacturer”

111
create or replace
trigger PROD_MAN_CHECK
before insert or update on PRODUCT
for each row
declare
l_man_id number;
begin
select man_id
into l_man_id
from manufacturer
where man_id = :new.man_id;
exception
when no_data_found then
raise_application_error(-20000,
'Manufacturer is not valid');
end;
every time we
create/change a
product...
...make sure the
manufacturer that
is provided already
exists

112
create or replace
trigger MAN_PROD_CHECK
before delete on MANUFACTURER
for each row
declare
l_prd_cnt number;
begin
select count(*)
into l_prd_cnt
from product
where man_id = :old.man_id;
if l_prd_cnt > 0 then
raise_application_error(-20001,
'Manufacturer in use by product records');
end if;
end;
before you can
delete a manufacturer...
...no products
can belong to it

114
SQL> insert into PRODUCT (...,MAN_ID,...) values (...,100, ...);
1 row created.
SQL> insert into PRODUCT (...,MAN_ID,...) values (...,101,...);
ERROR at line 1:
ORA-20000: Manufacturer is not valid
SQL> delete from MANUFACTURER where man_id = 5768;
ERROR at line 1:
ORA-20001: Manufacturer in use by product records

116
An unexpected error has
occurred. Manufacturer
record not found – orphan
product records!

117
SQL> select count(*)
2 from
3 ( select man_id from PRODUCT
4 minus
5 select man_id from MANUFACTURER )
6 /
COUNT(*)
----------
86

118
DIY integrity = no integrity

119
create PRD with MAN=7
Trigger fires
- does MAN=7 exist?
- "Yes"
- OK
delete MAN=7
Trigger fires
- any PRD with MAN=7
- "No" (not yet)
- OK
commit
commit

120
referential integrity seems simple…
“look up X before allowing Y”

121
referential integrity is complex
read consistency,
blocking / locking

122
I know what you're thinking ...

123
“we would never do anything that stupid”

“every hour,
populate secondary database
with a copy of the primary data”
EMP DWEMP
demo1a/b

127
“what percentage data integrity do we have?”
- project manager

128
concurrency testing vital
awesome for us as devs :-)

“All that integrity stuff is
a DBA issue, I just wanna
make sure my code runs fast”
demo3

132
it's not just adding constraints

134
“pseudo” relationships

135
sequencing issues
consider deferred constraints

136
beware stealth integrity rules

137
the "dreaded" module specification
"When creating a product...
validate registration date – ensure not in the future,
validate …
validate …
validate manufacturer ID – must exist in Manufacturer table"

Oracle consistency model is …

hand coding inter-table checking is
hard to do right

hand coding intra-table checking is
hard to do right
142

while we're talking R.I …
144

common mistake #3
145
poor performance by ignorance

149
"talking" to your database
... makes it faster

select prod_id, max(amount)
from stores st,
customers c,
sales s
where s.cust_id = c.cust_id(+)
and c.store_id = st.store_id
and s.amount > 10
group by prod_id
152
hash outer join ? nested loop ?
STORES first ?
sort merge ?

---------------------------------------------------
| Id | Operation | Name | Rows |
---------------------------------------------------
| 0 | SELECT STATEMENT | | 100 |
| 1 | HASH GROUP BY | | 100 |
|* 2 | HASH JOIN | | 990K |
| 3 | NESTED LOOPS SEMI | | 5000 |
| 4 | TABLE ACCESS FULL| CUSTOMERS | 5000 |
|* 5 | INDEX UNIQUE SCAN| STORE_IX | 50 |
|* 6 | TABLE ACCESS FULL | SALES | 990K |
---------------------------------------------------
153

154
can we do better ?
add indexes ?
rewrite query ?
result cache ?
materialized view ?

155
share your knowledge with the db
oc04.sql

common mistake #4
156
they aren't rows

a quick primer on (Oracle) memory

memory access = controlled access

SGA
protected by
2) access memory

SGA
protected by
3) release latch

SGA
protected by
someone must wait ...

spinning
can I have the latch ?
173

you ... get ... nothing ... done

nowadays...
yield
mutex
posting
Good reading: http://andreynikolaev.wordpress.com/
CAS

“But isn't that all internals
that are out of my control?”

to read data, you read blocks...

from disk
186
to memory
to you

SQL> select *
2 from EMP
3 where ...
/d01/ts/my_data1.dat
Buffer Cache

disks are s l o w
memory is fast

but memory access is complex
190
multi-process

SQL> select *
2 from EMP
3 where …
SQL> update EMP
2 set …
3 where …
SQL> select *
2 from EMP
3 where …
SQL> insert into
2 EMP
3 values (…)
SQL> select *
2 from CUSTOMER
3 /
SQL> select max(EMP)
2 from DEPT
SQL> delete
2 from DEPT
3 /

• Get latch
• Search along list of blocks in memory (or "chain")
• Read block
• Extract row of interest
• Release latch
• Give row back to client
"give me a row"
194

OPEN c_emp_list
LOOP
FETCH c_emp_list INTO ...
insert into EMP_COPY
( . . . )
values
( . . . );
END LOOP;
• Get latch
• Walk along chain
• Get block
• Extract single row
• Release latch
• Give row back to client
• Get latch (again)
• Walk along chain (again)
• Get block (again)
• Release latch (again)
• Give row back to client (again)
Row #1
Row #2
• Get latch (again)
• Walk along chain (again)
• Get block (again)
• Release latch (again)
• Give row back to client (again)
Row #3
198

a better strategy.... “pinning”

"give me a row"
• Get latch
• Get block
• Pin the block
• Release latch
• Give row 1 back to client
• Give row 2 back to client
…
• Give row n to client
• Get latch
• Remove my pin on the block
• Release latch
“by the way, I’ll be
wanting several
rows from this
block”

how to convey
“I may want many rows”

OracleConnection conn =
DriverManager.getConnection(
"jdbc:oracle:thin:@myhost:1521:mydb",
"username",
"password");
conn.SetDefaultRowPrefetch(100);

OracleCommand cmd = con.CreateCommand();
cmd.CommandText = "select * from employees";
OracleDataReader reader = cmd.ExecuteReader();
reader.FetchSize = cmd.RowSize * 100;
while (reader.Read())
;
reader.Dispose();

“relational”
not
“row-lational”
205

SQL> declare
2 n number;
3 d date;
4 cursor c is
5 select empno, hiredate
6 from big_table;
7 begin
8 open c;
9 loop
10 fetch c into n,d;
11 exit when c%notfound;
12 end loop;
13 close c;
14 end;
15 /
Elapsed: 00:01:15.06
1,000,000 rows

SQL> declare
2 cursor c is
3 select empno, hiredate
4 from big_table;
5 type c_list is table of c%rowtype;
6 r c_list;
7 begin
8 open c;
9 fetch c
10 bulk collect into r;
11 close c;
12 end;
13 /
Elapsed: 00:00:10.02

all that data goes somewhere
209

SQL> select * from v$mystats
2 where name like '%pga%'
3 /
NAME VALUE
------------------------------ ----------
session pga memory 101534672
session pga memory max 101534672

SQL> declare
. . .
9 begin
10 open c;
11 loop
12 fetch c
13 bulk collect
14 into r limit 100;
15 exit when c%notfound;
. . .
Elapsed: 00:00:08.03
NAME VALUE
------------------------------ ----------
session pga memory 225912
session pga memory max 225912

SQL> begin
2 for i in 1 .. 200000 loop
3 insert into t1 values ( i );
4 end loop;
5 end;
6 /
Elapsed: 00:00:11.03
217

SQL> declare
2 type num_list is table of number;
3 n num_list := num_list();
4 begin
5 n.extend(200000);
6 for i in 1 .. 200000 loop
7 n(i) := i;
8 end loop;
9 forall i in 1 .. 200000
10 insert into t1 values ( n(i) );
11 end;
12 /
Elapsed: 00:00:00.56

224
test, test, test, test
be brutal !

226
share the pain
at least internally

I love your feedback!
Rate your trip with Connor

Follow and subscribe
youtube bit.ly/youtube-connor
blog bit.ly/blog-connor
twitter bit.ly/twitter-connor
400+ posts mainly on database & development
250 technical videos, new uploads every week
rants and raves on tech and the world :-)

OpenWorld 2018 - Common Application Developer Disasters

More Related Content

What's hot

Similar to OpenWorld 2018 - Common Application Developer Disasters

More from Connor McDonald

Recently uploaded

OpenWorld 2018 - Common Application Developer Disasters