Skip to content

GitLab

Commit c6026618 authored by Craig Earley's avatar Craig Earley
Browse files

split sql script for modularity, added types and categories, still needs QA

parent 27a4cdb8
Hide whitespace changes
Inline Side-by-side
911_populate.sql 0 → 100644
View file @ c6026618
begin;
-- -- Create the tables
-- create table calls_csv (call_id char(15), date date, time time without time zone, complaint varchar, address varchar, city varchar, lat real, long real );
-- create table response_csv (call_id char(15), abbrev varchar, name varchar);
-- -- Populate them from the raw and uncleaned csv
\copy calls_csv from '/cluster/home/datascience/911calls/raw/2015calls.csv' with delimiter',' csv header;
\copy response_csv from '/cluster/home/datascience/911calls/raw/departmentsdispatched.csv' with delimiter',' csv;
-- set to initcap so as to make life easier
update calls_csv set city=initcap(city);
-- -- do something to get the additional geocodes we need: postgis? clean up, geocode
-- -- Update all the things
-- update agencies set agency_type = 'law' where agency_name like '%POLICE%';
-- update agencies set agency_type = 'fire' where agency_name like '%FIRE%';
-- update agencies set agency_type = 'law' where agency_name like '%SHERIFF%';
-- Make cities consistent and correct
update calls_csv set city= 'Richmond' where city like 'Rich%';
update calls_csv set city= 'Richmond' where city like '%chmond';
update calls_csv set city= 'Greens Fork' where city like 'Green%ork';
update calls_csv set city= 'Hagerstown' where city like 'Hage%';
update calls_csv set city= 'Boston' where city like 'Bost%';
update calls_csv set city= 'Connersville' where city like 'Connor%';
update calls_csv set city= 'Cambridge City' where city like 'Cambr%';
update calls_csv set city= 'New Paris' where city like 'New Paris%';
update calls_csv set city= 'Union City' where city like 'Union%';
update calls_csv set city= 'Randolph County' where city like 'Rando%';
update calls_csv set city= 'Fayette County' where city like 'Fayette%';
update calls_csv set city= 'Richmond' where city like '1 S 9th%';
update calls_csv set city= 'Economy' where city like 'Econ%';
update calls_csv set city= 'Centerville' where city like 'Cnet';
update calls_csv set city= 'Richmond' where city= 'Q';
update calls_csv set city= 'Richmond' where city= 'Wtt861';
update calls_csv set city= 'Farmland Area' where city like 'Farmland%';
update calls_csv set city= 'Richmond' where address like 'RPD';
update calls_csv set city= 'Richmond' where address like '18 N 22ND ST' and city like ' %';
-- Some things that are generic enough to update to Wayne County alone
update calls_csv set city= 'Wayne County' where city= 'Your City';
update calls_csv set city= 'Wayne County' where city= 'Unk';
update calls_csv set city= 'Wayne County' where city= 'Pu';
-- Very specific problems
update calls_csv set city= 'Richmond' where address like '%SW 18TH%';
update calls_csv set address= 'Bertschland, 1 E Church St' where city= '1 E Church St';
update calls_csv set city= 'Cambridge City' where city= '1 E Church St';
-- -- Modify the response_csv table
update response_csv set name=initcap(name);
update response_csv set name= 'Negotiators' where name like 'Negg%';
-- -- if no errors, insert the clean data into the appropriate postgres tables
-- -- spread the data downstream to the derived tables
-- Take care of calls_csv
-- want to use property cards for the ID over time
-- create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/geocodes.csv' with delimiter',' csv header;
insert into place (addr, lat, long, city) select distinct address, lat, long, city from calls_csv;
-- -- locations are processed, now use that
-- create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
-- fix this one:
insert into calls (call_id, place_id, date, time) select calls_csv.call_id, place.place_id, calls_csv.date, calls_csv.time from calls_csv inner join place on place.addr = calls_csv.address;
-- debug
-- create view address_align as select calls.call_id, calls.place_id, place.addr, place.place_id as place from calls inner join place on calls.place_id=place.place_id;
-- -- incident hierarchy
-- create table call_type (complaint_name varchar, type_name char(15), category_name varchar, category_id real);
insert into call_type (complaint_name, type_name) select distinct upper(complaint), upper(complaint) from calls_csv;
-- -- clean up the complaints
update call_type set type_name = '10-15' where complaint_name like '10-15%';
update call_type set type_name = '10-16' where complaint_name like '10-16%';
update call_type set type_name = '10-31' where complaint_name like '10-31%';
update call_type set type_name = '10-37' where complaint_name like '10-37%';
update call_type set type_name = '10-50' where complaint_name like '10-50%';
update call_type set type_name = '10-52' where complaint_name like '10-52%';
update call_type set type_name = '10-53' where complaint_name like '10-53%';
update call_type set type_name = '10-58' where complaint_name like '10-58%';
update call_type set type_name = '10-59' where complaint_name like '10-59%';
update call_type set type_name = '10-70' where complaint_name like '10-70%';
update call_type set type_name = '10-97' where complaint_name like '10-97%';
update call_type set type_name = '10-11' where complaint_name like '10-11%';
update call_type set type_name = '10-90' where complaint_name like '10-90%';
-- create table call_category (category_id serial unique, category_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- A giant block of redundant-looking code that picks category names
update call_type set category_name = 'law' where type_name like '10-10%';
update call_type set category_name = 'law' where type_name like '10-11%';
update call_type set category_name = 'law' where type_name like '10-14%';
update call_type set category_name = 'law' where type_name like '10-15%';
update call_type set category_name = 'law' where type_name like '10-16%';
update call_type set category_name = 'law' where type_name like '10-31%';
update call_type set category_name = 'law' where type_name like '10-32%';
update call_type set category_name = 'law' where type_name like '10-37%';
update call_type set category_name = 'law' where type_name like '10-38%';
update call_type set category_name = 'law' where type_name like '10-45%';
update call_type set category_name = 'law' where type_name like '10-53%';
update call_type set category_name = 'law' where type_name like '10-55%';
update call_type set category_name = 'law' where type_name like '10-56%';
update call_type set category_name = 'law' where type_name like '10-58%';
update call_type set category_name = 'law' where type_name like '10-59%';
update call_type set category_name = 'law' where type_name like '10-70%';
update call_type set category_name = 'law' where type_name like '10-90%';
update call_type set category_name = 'law' where type_name like '10-94%';
update call_type set category_name = 'law' where type_name like '10-96%';
update call_type set category_name = 'law' where type_name like '10-97%';
update call_type set category_name = 'law' where type_name like 'ARTICLE RECV%';
update call_type set category_name = 'law' where type_name like 'AUDIBLE ALRM%';
update call_type set category_name = 'law' where type_name like 'BAR CHECK%';
update call_type set category_name = 'law' where type_name like 'BIKE PICKUP%';
update call_type set category_name = 'law' where type_name like 'BUSINESS ALM%';
update call_type set category_name = 'law' where type_name like 'BUSINESS CK%';
update call_type set category_name = 'law' where type_name like 'CHILD ABUSE%';
update call_type set category_name = 'law' where type_name like 'CHILD NEGLEC%';
update call_type set category_name = 'law' where type_name like 'DRIVE OFF%';
update call_type set category_name = 'law' where type_name like 'EMER DETENT%';
update call_type set category_name = 'law' where type_name like 'EXTRA ATTEN%';
update call_type set category_name = 'law' where type_name like 'FALSE INFORM%';
update call_type set category_name = 'law' where type_name like 'FLAGGED DOWN%';
update call_type set category_name = 'law' where type_name like 'FOUND ARTCLE%';
update call_type set category_name = 'law' where type_name like 'GRAFFITI%';
update call_type set category_name = 'law' where type_name like 'HARASSMENT%';
update call_type set category_name = 'law' where type_name like 'HOLDUP ALARM%';
update call_type set category_name = 'law' where type_name like 'HOMICIDE%';
update call_type set category_name = 'law' where type_name like 'INCORIG JUV%';
update call_type set category_name = 'law' where type_name like 'INV%';
update call_type set category_name = 'law' where type_name like 'INV BAT ADLT%';
update call_type set category_name = 'law' where type_name like 'INV BAT JUV%';
update call_type set category_name = 'law' where type_name like 'INV B&E BUS%';
update call_type set category_name = 'law' where type_name like 'INV B&E OTH%';
update call_type set category_name = 'law' where type_name like 'INV B&E RES%';
update call_type set category_name = 'law' where type_name like 'INV CRIM REC%';
update call_type set category_name = 'law' where type_name like 'INV FORGERY%';
update call_type set category_name = 'law' where type_name like 'INV FRAUD%';
update call_type set category_name = 'law' where type_name like 'INV INVASION%';
update call_type set category_name = 'law' where type_name like 'INV MISCHIEF%';
update call_type set category_name = 'law' where type_name like 'INV ROBBERY%';
update call_type set category_name = 'law' where type_name like 'INV THEFT%';
update call_type set category_name = 'law' where type_name like 'INV VEH THFT%';
update call_type set category_name = 'law' where type_name like 'JUV PROBLEM%';
update call_type set category_name = 'law' where type_name like 'KIDNAP/ABDUC%';
update call_type set category_name = 'law' where type_name like 'LOCK OUT%';
update call_type set category_name = 'law' where type_name like 'LOST PLATE%';
update call_type set category_name = 'law' where type_name like 'MISSING PER%';
update call_type set category_name = 'law' where type_name like 'OPEN BUILDNG%';
update call_type set category_name = 'law' where type_name like 'OPEN VEHICLE%';
update call_type set category_name = 'law' where type_name like 'PEACE OFFICR%';
update call_type set category_name = 'law' where type_name like 'PROSTITUTION%';
update call_type set category_name = 'law' where type_name like 'PURSE SNATCH%';
update call_type set category_name = 'law' where type_name like 'PURSUIT FOOT%';
update call_type set category_name = 'law' where type_name like 'PURSUIT VEH%';
update call_type set category_name = 'law' where type_name like 'RECKLESS DRV%';
update call_type set category_name = 'law' where type_name like 'RESIDENTL AL%';
update call_type set category_name = 'law' where type_name like 'ROAD RAGE%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY RET%';
update call_type set category_name = 'law' where type_name like 'SEARCH WRRNT%';
update call_type set category_name = 'law' where type_name like 'SEX OFFENDER%';
update call_type set category_name = 'law' where type_name like 'SHOOTING%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER A%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER J%';
update call_type set category_name = 'law' where type_name like 'SHOTS FIRED%';
update call_type set category_name = 'law' where type_name like 'SIG 23%';
update call_type set category_name = 'law' where type_name like 'SIG 27%';
update call_type set category_name = 'law' where type_name like 'SIG 60%';
update call_type set category_name = 'law' where type_name like 'SIREN%';
update call_type set category_name = 'law' where type_name like 'SLIDE OFF%';
update call_type set category_name = 'law' where type_name like 'SPEAK W/OFF%';
update call_type set category_name = 'law' where type_name like 'SPECIAL EVNT%';
update call_type set category_name = 'law' where type_name like 'STABBING%';
update call_type set category_name = 'law' where type_name like 'STALKING%';
update call_type set category_name = 'law' where type_name like 'SUICIDAL SUB%';
update call_type set category_name = 'law' where type_name like 'TRAFFIC DTL%';
update call_type set category_name = 'law' where type_name like 'Traffic Stop%';
update call_type set category_name = 'law' where type_name like 'TRAFFIC STOP%';
update call_type set category_name = 'law' where type_name like 'TRANSPORT%';
update call_type set category_name = 'law' where type_name like 'TRASH DUMPNG%';
update call_type set category_name = 'law' where type_name like 'TREE DOWN%';
update call_type set category_name = 'law' where type_name like 'TRESPASSING%';
update call_type set category_name = 'law' where type_name like 'TRUANT%';
update call_type set category_name = 'law' where type_name like ' UNAUTH CNTRL%';
update call_type set category_name = 'law' where type_name like 'UNKNOWN PROB%';
update call_type set category_name = 'law' where type_name like 'VACANT HOUSE%';
update call_type set category_name = 'law' where type_name like 'VEHICLE RECV%';
update call_type set category_name = 'law' where type_name like 'VIN%';
update call_type set category_name = 'law' where type_name like 'WELFARE CK%';
update call_type set category_name = 'fire' where type_name like '10-37 ODOR%';
update call_type set category_name = 'fire' where type_name like '10-50 FATAL%';
update call_type set category_name = 'fire' where type_name like '10-50 PI H&R%';
update call_type set category_name = 'fire' where type_name like '10-50 UNK%';
update call_type set category_name = 'fire' where type_name like '10-52 BURN%';
update call_type set category_name = 'fire' where type_name like '10-52 ELECTR%';
update call_type set category_name = 'fire' where type_name like '10-70%';
update call_type set category_name = 'fire' where type_name like '10-70 CARBON%';
update call_type set category_name = 'fire' where type_name like '10-70 DUMP%';
update call_type set category_name = 'fire' where type_name like '10-70 GAS%';
update call_type set category_name = 'fire' where type_name like '10-70 GRASS%';
update call_type set category_name = 'fire' where type_name like '10-70 IL BRN%';
update call_type set category_name = 'fire' where type_name like '10-70 SMOKE%';
update call_type set category_name = 'fire' where type_name like '10-70 STRUCT%';
update call_type set category_name = 'fire' where type_name like '10-70 TRASH%';
update call_type set category_name = 'fire' where type_name like '10-70 VEHICL%';
update call_type set category_name = 'fire' where type_name like 'ARSON%';
update call_type set category_name = 'fire' where type_name like 'BURN%';
update call_type set category_name = 'fire' where type_name like 'FIREWORKS%';
update call_type set category_name = 'fire' where type_name like 'HAZMAT%';
update call_type set category_name = 'fire' where type_name like 'WIRES DOWN%';
-- -- Maybe deal with these on their own terms
-- update call_type set category_name = 'misc' where type_name like '10-17%';
-- update call_type set category_name = 'misc' where type_name like '10-53%';
-- update call_type set category_name = 'misc' where type_name like '10-53 HC%';
-- update call_type set category_name = 'misc' where type_name like '10-54%';
-- update call_type set category_name = 'misc' where type_name like '10-59%';
-- update call_type set category_name = 'misc' where type_name like 'APS%';
-- update call_type set category_name = 'misc' where type_name like 'ASSIST OTHER%';
-- update call_type set category_name = 'misc' where type_name like 'BIKE%';
-- update call_type set category_name = 'misc' where type_name like 'GENERAL%';
-- update call_type set category_name = 'misc' where type_name like 'HOTEL/MOTEL%';
-- update call_type set category_name = 'misc' where type_name like 'MCAB%';
-- update call_type set category_name = 'misc' where type_name like 'MSC%';
-- update call_type set category_name = 'misc' where type_name like 'PERSNL ASST%';
-- update call_type set category_name = 'misc' where type_name like 'P&W%';
-- update call_type set category_name = 'misc' where type_name like 'REPO%';
-- update call_type set category_name = 'misc' where type_name like 'SIG 17%';
-- update call_type set category_name = 'misc' where type_name like 'UNWANTED%';
update call_type set category_name = 'ems' where type_name like '10-52%';
update call_type set category_name = 'ems' where type_name like '10-52 ALLERG%';
update call_type set category_name = 'ems' where type_name like '10-52 BIRTH%';
update call_type set category_name = 'ems' where type_name like '10-52 CP%';
update call_type set category_name = 'ems' where type_name like '10-52 DIABET%';
update call_type set category_name = 'ems' where type_name like '10-52 FALL%';
update call_type set category_name = 'ems' where type_name like '10-52 HEAD%';
update call_type set category_name = 'ems' where type_name like '10-52 HEART%';
update call_type set category_name = 'ems' where type_name like '10-52 LIFT%';
update call_type set category_name = 'ems' where type_name like '10-52 SEIZU%';
update call_type set category_name = 'ems' where type_name like '10-52 STROKE%';
update call_type set category_name = 'ems' where type_name like '10-52 TRANS%';
update call_type set category_name = 'ems' where type_name like '10-52 UNK%';
update call_type set category_name = 'ems' where type_name like 'IND EXPOSURE%';
update call_type set category_name = 'ems' where type_name like 'MEDICAL ALRM%';
update call_type set category_name = 'ems' where type_name like 'OVERDOSE%';
update call_type set category_name = 'misc' where category_name is null;
--update call_type set category_id = call_category.category_id from call_category where call_category.category_name=call_type.category_name;
--UPDATE C SET column1 = A.column1,column2 = B.column2,column3 = A.column1 + B.column2 FROM A JOIN B ON A.id = B.id WHERE C.id = A.id AND (C.column1, C.column2, C.column3) IS DISTINCT FROM (A.column1, B.column2, A.column1 + B.column2);
insert into call_category (category_name) select distinct category_name from call_type;
-- create table call_group (group_id real, group_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- Now take care of response_csv
-- -- agency processing: create the tables, populate the agency table, and from those id's populate the response table
-- create table agency (agency_id serial unique, agency_abbrev varchar, agency_name varchar, city varchar);
insert into agency (agency_abbrev, agency_name) select distinct abbrev,name from response_csv;
-- create table agency_type (agency_id serial unique, type_name char(15));
-- create table response (call_id char(15), agency_id real);
insert into response (call_id, agency_id) select calls_csv.call_id, agency.agency_id from calls_csv inner join response_csv on response_csv.call_id=calls_csv.call_id inner join agency on response_csv.abbrev=agency.agency_abbrev;
update call_type set category_id = call_category.category_id from call_category where call_category.category_name = call_type.category_name;
commit;
911_table_creation.sql deleted 100644 → 0
View file @ 27a4cdb8
begin;
-- -- Create the tables
create table calls_csv (call_id char(15), date date, time time without time zone, complaint varchar, address varchar, city varchar, lat real, long real );
create table response_csv (call_id char(15), abbrev varchar, name varchar);
-- -- Populate them from the raw and uncleaned csv
\copy calls_csv from '/cluster/home/datascience/911calls/raw/2015calls.csv' with delimiter ',' csv header;
\copy response_csv from '/cluster/home/datascience/911calls/raw/departmentsdispatched.csv' with delimiter ',' csv;
-- set to initcap so as to make life easier
update calls_csv set city=initcap(city);
-- -- do something to get the additional geocodes we need: postgis? clean up, geocode
-- -- Update all the things
-- update agencies set agency_type = 'law' where agency_name like '%POLICE%';
-- update agencies set agency_type = 'fire' where agency_name like '%FIRE%';
-- update agencies set agency_type = 'law' where agency_name like '%SHERIFF%';
-- Make cities consistent and correct
update calls_csv set city='Richmond' where city like 'Rich%';
update calls_csv set city='Richmond' where city like '%chmond';
update calls_csv set city='Greens Fork' where city like 'Green%ork';
update calls_csv set city='Hagerstown' where city like 'Hage%';
update calls_csv set city='Boston' where city like 'Bost%';
update calls_csv set city='Connersville' where city like 'Connor%';
update calls_csv set city='Cambridge City' where city like 'Cambr%';
update calls_csv set city='New Paris' where city like 'New Paris%';
update calls_csv set city='Union City' where city like 'Union%';
update calls_csv set city='Randolph County' where city like 'Rando%';
update calls_csv set city='Fayette County' where city like 'Fayette%';
update calls_csv set city='Richmond' where city like '1 S 9th%';
update calls_csv set city='Economy' where city like 'Econ%';
update calls_csv set city='Centerville' where city like 'Cnet';
update calls_csv set city='Richmond' where city='Q';
update calls_csv set city='Richmond' where city='Wtt861';
update calls_csv set city='Farmland Area' where city like 'Farmland%';
update calls_csv set city='Richmond' where address like 'RPD';
update calls_csv set city='Richmond' where address like '18 N 22ND ST' and city like ' %';
-- Some things that are generic enough to update to Wayne County alone
update calls_csv set city='Wayne County' where city='Your City';
update calls_csv set city='Wayne County' where city='Unk';
update calls_csv set city='Wayne County' where city='Pu';
-- Very specific problems
update calls_csv set city='Richmond' where address like '%SW 18TH%';
update calls_csv set address= 'Bertschland, 1 E Church St' where city='1 E Church St';
update calls_csv set city='Cambridge City' where city='1 E Church St';
-- -- Modify the response_csv table
update response_csv set name=initcap(name);
update response_csv set name='Negotiators' where name like 'Negg%';
-- -- if no errors, insert the clean data into the appropriate postgres tables
-- -- spread the data downstream to the derived tables
-- -- Take care of calls_csv
-- want to use property cards for the ID over time
create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/geocodes.csv' with delimiter ',' csv header;
insert into place (addr, lat, long, city) select distinct address, lat, long, city from calls_csv;
-- -- locations are processed, now use that
create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
-- fix this one:
insert into calls (call_id, place_id, date, time) select calls_csv.call_id, place.place_id, calls_csv.date, calls_csv.time from calls_csv inner join place on place.addr = calls_csv.address;
-- debug
create view address_align as select calls.call_id, calls.place_id, place.addr, place.place_id as place from calls inner join place on calls.place_id=place.place_id;
-- -- incident hierarchy
create table call_type (complaint_name varchar, type_name char(15), category_id char(15));
insert into call_type (complaint_name, type_name) select distinct upper(complaint), upper(complaint) from calls_csv;
-- -- clean up the complaints
update call_type set type_name = '10-15' where complaint_name like '10-15%';
update call_type set type_name = '10-16' where complaint_name like '10-16%';
update call_type set type_name = '10-31' where complaint_name like '10-31%';
update call_type set type_name = '10-37' where complaint_name like '10-37%';
update call_type set type_name = '10-50' where complaint_name like '10-50%';
update call_type set type_name = '10-52' where complaint_name like '10-52%';
update call_type set type_name = '10-53' where complaint_name like '10-53%';
update call_type set type_name = '10-58' where complaint_name like '10-58%';
update call_type set type_name = '10-59' where complaint_name like '10-59%';
update call_type set type_name = '10-70' where complaint_name like '10-70%';
update call_type set type_name = '10-97' where complaint_name like '10-97%';
update call_type set type_name = '10-11' where complaint_name like '10-11%';
update call_type set type_name = '10-90' where complaint_name like '10-90%';
create table call_category (category_id serial unique, category_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
create table call_group (group_id real, group_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- Now take care of response_csv
-- -- agency processing: create the tables, populate the agency table, and from those id's populate the response table
create table agency (agency_id serial unique, agency_abbrev varchar, agency_name varchar, city varchar);
insert into agency (agency_abbrev, agency_name) select distinct abbrev,name from response_csv;
create table agency_type (agency_id serial unique, type_name char(15));
create table response (call_id char(15), agency_id real);
insert into response (call_id, agency_id) select calls_csv.call_id, agency.agency_id from calls_csv inner join response_csv on response_csv.call_id=calls_csv.call_id inner join agency on response_csv.abbrev=agency.agency_abbrev;
commit;
911_tables.sql 0 → 100644
View file @ c6026618
begin;
-- -- Create the tables
create table calls_csv (call_id char(15), date date, time time without time zone, complaint varchar, address varchar, city varchar, lat real, long real );
create table response_csv (call_id char(15), abbrev varchar, name varchar);
-- -- Populate them from the raw and uncleaned csv
\copy calls_csv from '/cluster/home/datascience/911calls/raw/2015calls.csv' with delimiter',' csv header;
\copy response_csv from '/cluster/home/datascience/911calls/raw/departmentsdispatched.csv' with delimiter',' csv;
-- set to initcap so as to make life easier
update calls_csv set city=initcap(city);
-- -- do something to get the additional geocodes we need: postgis? clean up, geocode
-- -- Update all the things
-- update agencies set agency_type = 'law' where agency_name like '%POLICE%';
-- update agencies set agency_type = 'fire' where agency_name like '%FIRE%';
-- update agencies set agency_type = 'law' where agency_name like '%SHERIFF%';
-- Make cities consistent and correct
update calls_csv set city= 'Richmond' where city like 'Rich%';
update calls_csv set city= 'Richmond' where city like '%chmond';
update calls_csv set city= 'Greens Fork' where city like 'Green%ork';
update calls_csv set city= 'Hagerstown' where city like 'Hage%';
update calls_csv set city= 'Boston' where city like 'Bost%';
update calls_csv set city= 'Connersville' where city like 'Connor%';
update calls_csv set city= 'Cambridge City' where city like 'Cambr%';
update calls_csv set city= 'New Paris' where city like 'New Paris%';
update calls_csv set city= 'Union City' where city like 'Union%';
update calls_csv set city= 'Randolph County' where city like 'Rando%';
update calls_csv set city= 'Fayette County' where city like 'Fayette%';
update calls_csv set city= 'Richmond' where city like '1 S 9th%';
update calls_csv set city= 'Economy' where city like 'Econ%';
update calls_csv set city= 'Centerville' where city like 'Cnet';
update calls_csv set city= 'Richmond' where city= 'Q';
update calls_csv set city= 'Richmond' where city= 'Wtt861';
update calls_csv set city= 'Farmland Area' where city like 'Farmland%';
update calls_csv set city= 'Richmond' where address like 'RPD';
update calls_csv set city= 'Richmond' where address like '18 N 22ND ST' and city like ' %';
-- Some things that are generic enough to update to Wayne County alone
update calls_csv set city= 'Wayne County' where city= 'Your City';
update calls_csv set city= 'Wayne County' where city= 'Unk';
update calls_csv set city= 'Wayne County' where city= 'Pu';
-- Very specific problems
update calls_csv set city= 'Richmond' where address like '%SW 18TH%';
update calls_csv set address= 'Bertschland, 1 E Church St' where city= '1 E Church St';
update calls_csv set city= 'Cambridge City' where city= '1 E Church St';
-- -- Modify the response_csv table
update response_csv set name=initcap(name);
update response_csv set name= 'Negotiators' where name like 'Negg%';
-- -- if no errors, insert the clean data into the appropriate postgres tables
-- -- spread the data downstream to the derived tables
-- -- Take care of calls_csv
-- want to use property cards for the ID over time
create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/geocodes.csv' with delimiter',' csv header;
insert into place (addr, lat, long, city) select distinct address, lat, long, city from calls_csv;
-- -- locations are processed, now use that
create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
-- fix this one:
insert into calls (call_id, place_id, date, time) select calls_csv.call_id, place.place_id, calls_csv.date, calls_csv.time from calls_csv inner join place on place.addr = calls_csv.address;
-- debug
create view address_align as select calls.call_id, calls.place_id, place.addr, place.place_id as place from calls inner join place on calls.place_id=place.place_id;
-- -- incident hierarchy
create table call_type (complaint_name varchar, type_name char(15), category_name varchar, category_id real);
insert into call_type (complaint_name, type_name) select distinct upper(complaint), upper(complaint) from calls_csv;
-- -- clean up the complaints
update call_type set type_name = '10-15' where complaint_name like '10-15%';
update call_type set type_name = '10-16' where complaint_name like '10-16%';
update call_type set type_name = '10-31' where complaint_name like '10-31%';
update call_type set type_name = '10-37' where complaint_name like '10-37%';
update call_type set type_name = '10-50' where complaint_name like '10-50%';
update call_type set type_name = '10-52' where complaint_name like '10-52%';
update call_type set type_name = '10-53' where complaint_name like '10-53%';
update call_type set type_name = '10-58' where complaint_name like '10-58%';
update call_type set type_name = '10-59' where complaint_name like '10-59%';
update call_type set type_name = '10-70' where complaint_name like '10-70%';
update call_type set type_name = '10-97' where complaint_name like '10-97%';
update call_type set type_name = '10-11' where complaint_name like '10-11%';
update call_type set type_name = '10-90' where complaint_name like '10-90%';
create table call_category (category_id serial unique, category_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- A giant block of redundant-looking code that picks category names
update call_type set category_name = 'law' where type_name like '10-10%';
update call_type set category_name = 'law' where type_name like '10-11%';
update call_type set category_name = 'law' where type_name like '10-14%';
update call_type set category_name = 'law' where type_name like '10-15%';
update call_type set category_name = 'law' where type_name like '10-16%';
update call_type set category_name = 'law' where type_name like '10-31%';
update call_type set category_name = 'law' where type_name like '10-32%';
update call_type set category_name = 'law' where type_name like '10-37%';
update call_type set category_name = 'law' where type_name like '10-38%';
update call_type set category_name = 'law' where type_name like '10-45%';
update call_type set category_name = 'law' where type_name like '10-53%';
update call_type set category_name = 'law' where type_name like '10-55%';
update call_type set category_name = 'law' where type_name like '10-56%';
update call_type set category_name = 'law' where type_name like '10-58%';
update call_type set category_name = 'law' where type_name like '10-59%';
update call_type set category_name = 'law' where type_name like '10-70%';
update call_type set category_name = 'law' where type_name like '10-90%';
update call_type set category_name = 'law' where type_name like '10-94%';
update call_type set category_name = 'law' where type_name like '10-96%';
update call_type set category_name = 'law' where type_name like '10-97%';
update call_type set category_name = 'law' where type_name like 'ARTICLE RECV%';
update call_type set category_name = 'law' where type_name like 'AUDIBLE ALRM%';
update call_type set category_name = 'law' where type_name like 'BAR CHECK%';
update call_type set category_name = 'law' where type_name like 'BIKE PICKUP%';
update call_type set category_name = 'law' where type_name like 'BUSINESS ALM%';
update call_type set category_name = 'law' where type_name like 'BUSINESS CK%';
update call_type set category_name = 'law' where type_name like 'CHILD ABUSE%';
update call_type set category_name = 'law' where type_name like 'CHILD NEGLEC%';
update call_type set category_name = 'law' where type_name like 'DRIVE OFF%';
update call_type set category_name = 'law' where type_name like 'EMER DETENT%';
update call_type set category_name = 'law' where type_name like 'EXTRA ATTEN%';
update call_type set category_name = 'law' where type_name like 'FALSE INFORM%';
update call_type set category_name = 'law' where type_name like 'FLAGGED DOWN%';
update call_type set category_name = 'law' where type_name like 'FOUND ARTCLE%';
update call_type set category_name = 'law' where type_name like 'GRAFFITI%';
update call_type set category_name = 'law' where type_name like 'HARASSMENT%';
update call_type set category_name = 'law' where type_name like 'HOLDUP ALARM%';
update call_type set category_name = 'law' where type_name like 'HOMICIDE%';
update call_type set category_name = 'law' where type_name like 'INCORIG JUV%';
update call_type set category_name = 'law' where type_name like 'INV%';
update call_type set category_name = 'law' where type_name like 'INV BAT ADLT%';
update call_type set category_name = 'law' where type_name like 'INV BAT JUV%';
update call_type set category_name = 'law' where type_name like 'INV B&E BUS%';
update call_type set category_name = 'law' where type_name like 'INV B&E OTH%';
update call_type set category_name = 'law' where type_name like 'INV B&E RES%';
update call_type set category_name = 'law' where type_name like 'INV CRIM REC%';
update call_type set category_name = 'law' where type_name like 'INV FORGERY%';
update call_type set category_name = 'law' where type_name like 'INV FRAUD%';
update call_type set category_name = 'law' where type_name like 'INV INVASION%';
update call_type set category_name = 'law' where type_name like 'INV MISCHIEF%';
update call_type set category_name = 'law' where type_name like 'INV ROBBERY%';
update call_type set category_name = 'law' where type_name like 'INV THEFT%';
update call_type set category_name = 'law' where type_name like 'INV VEH THFT%';
update call_type set category_name = 'law' where type_name like 'JUV PROBLEM%';
update call_type set category_name = 'law' where type_name like 'KIDNAP/ABDUC%';
update call_type set category_name = 'law' where type_name like 'LOCK OUT%';
update call_type set category_name = 'law' where type_name like 'LOST PLATE%';
update call_type set category_name = 'law' where type_name like 'MISSING PER%';
update call_type set category_name = 'law' where type_name like 'OPEN BUILDNG%';
update call_type set category_name = 'law' where type_name like 'OPEN VEHICLE%';
update call_type set category_name = 'law' where type_name like 'PEACE OFFICR%';
update call_type set category_name = 'law' where type_name like 'PROSTITUTION%';
update call_type set category_name = 'law' where type_name like 'PURSE SNATCH%';
update call_type set category_name = 'law' where type_name like 'PURSUIT FOOT%';
update call_type set category_name = 'law' where type_name like 'PURSUIT VEH%';
update call_type set category_name = 'law' where type_name like 'RECKLESS DRV%';
update call_type set category_name = 'law' where type_name like 'RESIDENTL AL%';
update call_type set category_name = 'law' where type_name like 'ROAD RAGE%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY RET%';
update call_type set category_name = 'law' where type_name like 'SEARCH WRRNT%';
update call_type set category_name = 'law' where type_name like 'SEX OFFENDER%';
update call_type set category_name = 'law' where type_name like 'SHOOTING%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER A%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER J%';
update call_type set category_name = 'law' where type_name like 'SHOTS FIRED%';
update call_type set category_name = 'law' where type_name like 'SIG 23%';
update call_type set category_name = 'law' where type_name like 'SIG 27%';
update call_type set category_name = 'law' where type_name like 'SIG 60%';
update call_type set category_name = 'law' where type_name like 'SIREN%';
update call_type set category_name = 'law' where type_name like 'SLIDE OFF%';
update call_type set category_name = 'law' where type_name like 'SPEAK W/OFF%';
update call_type set category_name = 'law' where type_name like 'SPECIAL EVNT%';
update call_type set category_name = 'law' where type_name like 'STABBING%';
update call_type set category_name = 'law' where type_name like 'STALKING%';
update call_type set category_name = 'law' where type_name like 'SUICIDAL SUB%';
update call_type set category_name = 'law' where type_name like 'TRAFFIC DTL%';