Commit c6026618 authored by Craig Earley's avatar Craig Earley
Browse files

split sql script for modularity, added types and categories, still needs QA

parent 27a4cdb8
begin;
-- -- Create the tables
-- create table calls_csv (call_id char(15), date date, time time without time zone, complaint varchar, address varchar, city varchar, lat real, long real );
-- create table response_csv (call_id char(15), abbrev varchar, name varchar);
-- -- Populate them from the raw and uncleaned csv
\copy calls_csv from '/cluster/home/datascience/911calls/raw/2015calls.csv' with delimiter',' csv header;
\copy response_csv from '/cluster/home/datascience/911calls/raw/departmentsdispatched.csv' with delimiter',' csv;
-- set to initcap so as to make life easier
update calls_csv set city=initcap(city);
-- -- do something to get the additional geocodes we need: postgis? clean up, geocode
-- -- Update all the things
-- update agencies set agency_type = 'law' where agency_name like '%POLICE%';
-- update agencies set agency_type = 'fire' where agency_name like '%FIRE%';
-- update agencies set agency_type = 'law' where agency_name like '%SHERIFF%';
-- Make cities consistent and correct
update calls_csv set city= 'Richmond' where city like 'Rich%';
update calls_csv set city= 'Richmond' where city like '%chmond';
update calls_csv set city= 'Greens Fork' where city like 'Green%ork';
update calls_csv set city= 'Hagerstown' where city like 'Hage%';
update calls_csv set city= 'Boston' where city like 'Bost%';
update calls_csv set city= 'Connersville' where city like 'Connor%';
update calls_csv set city= 'Cambridge City' where city like 'Cambr%';
update calls_csv set city= 'New Paris' where city like 'New Paris%';
update calls_csv set city= 'Union City' where city like 'Union%';
update calls_csv set city= 'Randolph County' where city like 'Rando%';
update calls_csv set city= 'Fayette County' where city like 'Fayette%';
update calls_csv set city= 'Richmond' where city like '1 S 9th%';
update calls_csv set city= 'Economy' where city like 'Econ%';
update calls_csv set city= 'Centerville' where city like 'Cnet';
update calls_csv set city= 'Richmond' where city= 'Q';
update calls_csv set city= 'Richmond' where city= 'Wtt861';
update calls_csv set city= 'Farmland Area' where city like 'Farmland%';
update calls_csv set city= 'Richmond' where address like 'RPD';
update calls_csv set city= 'Richmond' where address like '18 N 22ND ST' and city like ' %';
-- Some things that are generic enough to update to Wayne County alone
update calls_csv set city= 'Wayne County' where city= 'Your City';
update calls_csv set city= 'Wayne County' where city= 'Unk';
update calls_csv set city= 'Wayne County' where city= 'Pu';
-- Very specific problems
update calls_csv set city= 'Richmond' where address like '%SW 18TH%';
update calls_csv set address= 'Bertschland, 1 E Church St' where city= '1 E Church St';
update calls_csv set city= 'Cambridge City' where city= '1 E Church St';
-- -- Modify the response_csv table
update response_csv set name=initcap(name);
update response_csv set name= 'Negotiators' where name like 'Negg%';
-- -- if no errors, insert the clean data into the appropriate postgres tables
-- -- spread the data downstream to the derived tables
-- Take care of calls_csv
-- want to use property cards for the ID over time
-- create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/geocodes.csv' with delimiter',' csv header;
insert into place (addr, lat, long, city) select distinct address, lat, long, city from calls_csv;
-- -- locations are processed, now use that
-- create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
-- fix this one:
insert into calls (call_id, place_id, date, time) select calls_csv.call_id, place.place_id, calls_csv.date, calls_csv.time from calls_csv inner join place on place.addr = calls_csv.address;
-- debug
-- create view address_align as select calls.call_id, calls.place_id, place.addr, place.place_id as place from calls inner join place on calls.place_id=place.place_id;
-- -- incident hierarchy
-- create table call_type (complaint_name varchar, type_name char(15), category_name varchar, category_id real);
insert into call_type (complaint_name, type_name) select distinct upper(complaint), upper(complaint) from calls_csv;
-- -- clean up the complaints
update call_type set type_name = '10-15' where complaint_name like '10-15%';
update call_type set type_name = '10-16' where complaint_name like '10-16%';
update call_type set type_name = '10-31' where complaint_name like '10-31%';
update call_type set type_name = '10-37' where complaint_name like '10-37%';
update call_type set type_name = '10-50' where complaint_name like '10-50%';
update call_type set type_name = '10-52' where complaint_name like '10-52%';
update call_type set type_name = '10-53' where complaint_name like '10-53%';
update call_type set type_name = '10-58' where complaint_name like '10-58%';
update call_type set type_name = '10-59' where complaint_name like '10-59%';
update call_type set type_name = '10-70' where complaint_name like '10-70%';
update call_type set type_name = '10-97' where complaint_name like '10-97%';
update call_type set type_name = '10-11' where complaint_name like '10-11%';
update call_type set type_name = '10-90' where complaint_name like '10-90%';
-- create table call_category (category_id serial unique, category_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- A giant block of redundant-looking code that picks category names
update call_type set category_name = 'law' where type_name like '10-10%';
update call_type set category_name = 'law' where type_name like '10-11%';
update call_type set category_name = 'law' where type_name like '10-14%';
update call_type set category_name = 'law' where type_name like '10-15%';
update call_type set category_name = 'law' where type_name like '10-16%';
update call_type set category_name = 'law' where type_name like '10-31%';
update call_type set category_name = 'law' where type_name like '10-32%';
update call_type set category_name = 'law' where type_name like '10-37%';
update call_type set category_name = 'law' where type_name like '10-38%';
update call_type set category_name = 'law' where type_name like '10-45%';
update call_type set category_name = 'law' where type_name like '10-53%';
update call_type set category_name = 'law' where type_name like '10-55%';
update call_type set category_name = 'law' where type_name like '10-56%';
update call_type set category_name = 'law' where type_name like '10-58%';
update call_type set category_name = 'law' where type_name like '10-59%';
update call_type set category_name = 'law' where type_name like '10-70%';
update call_type set category_name = 'law' where type_name like '10-90%';
update call_type set category_name = 'law' where type_name like '10-94%';
update call_type set category_name = 'law' where type_name like '10-96%';
update call_type set category_name = 'law' where type_name like '10-97%';
update call_type set category_name = 'law' where type_name like 'ARTICLE RECV%';
update call_type set category_name = 'law' where type_name like 'AUDIBLE ALRM%';
update call_type set category_name = 'law' where type_name like 'BAR CHECK%';
update call_type set category_name = 'law' where type_name like 'BIKE PICKUP%';
update call_type set category_name = 'law' where type_name like 'BUSINESS ALM%';
update call_type set category_name = 'law' where type_name like 'BUSINESS CK%';
update call_type set category_name = 'law' where type_name like 'CHILD ABUSE%';
update call_type set category_name = 'law' where type_name like 'CHILD NEGLEC%';
update call_type set category_name = 'law' where type_name like 'DRIVE OFF%';
update call_type set category_name = 'law' where type_name like 'EMER DETENT%';
update call_type set category_name = 'law' where type_name like 'EXTRA ATTEN%';
update call_type set category_name = 'law' where type_name like 'FALSE INFORM%';
update call_type set category_name = 'law' where type_name like 'FLAGGED DOWN%';
update call_type set category_name = 'law' where type_name like 'FOUND ARTCLE%';
update call_type set category_name = 'law' where type_name like 'GRAFFITI%';
update call_type set category_name = 'law' where type_name like 'HARASSMENT%';
update call_type set category_name = 'law' where type_name like 'HOLDUP ALARM%';
update call_type set category_name = 'law' where type_name like 'HOMICIDE%';
update call_type set category_name = 'law' where type_name like 'INCORIG JUV%';
update call_type set category_name = 'law' where type_name like 'INV%';
update call_type set category_name = 'law' where type_name like 'INV BAT ADLT%';
update call_type set category_name = 'law' where type_name like 'INV BAT JUV%';
update call_type set category_name = 'law' where type_name like 'INV B&E BUS%';
update call_type set category_name = 'law' where type_name like 'INV B&E OTH%';
update call_type set category_name = 'law' where type_name like 'INV B&E RES%';
update call_type set category_name = 'law' where type_name like 'INV CRIM REC%';
update call_type set category_name = 'law' where type_name like 'INV FORGERY%';
update call_type set category_name = 'law' where type_name like 'INV FRAUD%';
update call_type set category_name = 'law' where type_name like 'INV INVASION%';
update call_type set category_name = 'law' where type_name like 'INV MISCHIEF%';
update call_type set category_name = 'law' where type_name like 'INV ROBBERY%';
update call_type set category_name = 'law' where type_name like 'INV THEFT%';
update call_type set category_name = 'law' where type_name like 'INV VEH THFT%';
update call_type set category_name = 'law' where type_name like 'JUV PROBLEM%';
update call_type set category_name = 'law' where type_name like 'KIDNAP/ABDUC%';
update call_type set category_name = 'law' where type_name like 'LOCK OUT%';
update call_type set category_name = 'law' where type_name like 'LOST PLATE%';
update call_type set category_name = 'law' where type_name like 'MISSING PER%';
update call_type set category_name = 'law' where type_name like 'OPEN BUILDNG%';
update call_type set category_name = 'law' where type_name like 'OPEN VEHICLE%';
update call_type set category_name = 'law' where type_name like 'PEACE OFFICR%';
update call_type set category_name = 'law' where type_name like 'PROSTITUTION%';
update call_type set category_name = 'law' where type_name like 'PURSE SNATCH%';
update call_type set category_name = 'law' where type_name like 'PURSUIT FOOT%';
update call_type set category_name = 'law' where type_name like 'PURSUIT VEH%';
update call_type set category_name = 'law' where type_name like 'RECKLESS DRV%';
update call_type set category_name = 'law' where type_name like 'RESIDENTL AL%';
update call_type set category_name = 'law' where type_name like 'ROAD RAGE%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY RET%';
update call_type set category_name = 'law' where type_name like 'SEARCH WRRNT%';
update call_type set category_name = 'law' where type_name like 'SEX OFFENDER%';
update call_type set category_name = 'law' where type_name like 'SHOOTING%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER A%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER J%';
update call_type set category_name = 'law' where type_name like 'SHOTS FIRED%';
update call_type set category_name = 'law' where type_name like 'SIG 23%';
update call_type set category_name = 'law' where type_name like 'SIG 27%';
update call_type set category_name = 'law' where type_name like 'SIG 60%';
update call_type set category_name = 'law' where type_name like 'SIREN%';
update call_type set category_name = 'law' where type_name like 'SLIDE OFF%';
update call_type set category_name = 'law' where type_name like 'SPEAK W/OFF%';
update call_type set category_name = 'law' where type_name like 'SPECIAL EVNT%';
update call_type set category_name = 'law' where type_name like 'STABBING%';
update call_type set category_name = 'law' where type_name like 'STALKING%';
update call_type set category_name = 'law' where type_name like 'SUICIDAL SUB%';
update call_type set category_name = 'law' where type_name like 'TRAFFIC DTL%';
update call_type set category_name = 'law' where type_name like 'Traffic Stop%';
update call_type set category_name = 'law' where type_name like 'TRAFFIC STOP%';
update call_type set category_name = 'law' where type_name like 'TRANSPORT%';
update call_type set category_name = 'law' where type_name like 'TRASH DUMPNG%';
update call_type set category_name = 'law' where type_name like 'TREE DOWN%';
update call_type set category_name = 'law' where type_name like 'TRESPASSING%';
update call_type set category_name = 'law' where type_name like 'TRUANT%';
update call_type set category_name = 'law' where type_name like ' UNAUTH CNTRL%';
update call_type set category_name = 'law' where type_name like 'UNKNOWN PROB%';
update call_type set category_name = 'law' where type_name like 'VACANT HOUSE%';
update call_type set category_name = 'law' where type_name like 'VEHICLE RECV%';
update call_type set category_name = 'law' where type_name like 'VIN%';
update call_type set category_name = 'law' where type_name like 'WELFARE CK%';
update call_type set category_name = 'fire' where type_name like '10-37 ODOR%';
update call_type set category_name = 'fire' where type_name like '10-50 FATAL%';
update call_type set category_name = 'fire' where type_name like '10-50 PI H&R%';
update call_type set category_name = 'fire' where type_name like '10-50 UNK%';
update call_type set category_name = 'fire' where type_name like '10-52 BURN%';
update call_type set category_name = 'fire' where type_name like '10-52 ELECTR%';
update call_type set category_name = 'fire' where type_name like '10-70%';
update call_type set category_name = 'fire' where type_name like '10-70 CARBON%';
update call_type set category_name = 'fire' where type_name like '10-70 DUMP%';
update call_type set category_name = 'fire' where type_name like '10-70 GAS%';
update call_type set category_name = 'fire' where type_name like '10-70 GRASS%';
update call_type set category_name = 'fire' where type_name like '10-70 IL BRN%';
update call_type set category_name = 'fire' where type_name like '10-70 SMOKE%';
update call_type set category_name = 'fire' where type_name like '10-70 STRUCT%';
update call_type set category_name = 'fire' where type_name like '10-70 TRASH%';
update call_type set category_name = 'fire' where type_name like '10-70 VEHICL%';
update call_type set category_name = 'fire' where type_name like 'ARSON%';
update call_type set category_name = 'fire' where type_name like 'BURN%';
update call_type set category_name = 'fire' where type_name like 'FIREWORKS%';
update call_type set category_name = 'fire' where type_name like 'HAZMAT%';
update call_type set category_name = 'fire' where type_name like 'WIRES DOWN%';
-- -- Maybe deal with these on their own terms
-- update call_type set category_name = 'misc' where type_name like '10-17%';
-- update call_type set category_name = 'misc' where type_name like '10-53%';
-- update call_type set category_name = 'misc' where type_name like '10-53 HC%';
-- update call_type set category_name = 'misc' where type_name like '10-54%';
-- update call_type set category_name = 'misc' where type_name like '10-59%';
-- update call_type set category_name = 'misc' where type_name like 'APS%';
-- update call_type set category_name = 'misc' where type_name like 'ASSIST OTHER%';
-- update call_type set category_name = 'misc' where type_name like 'BIKE%';
-- update call_type set category_name = 'misc' where type_name like 'GENERAL%';
-- update call_type set category_name = 'misc' where type_name like 'HOTEL/MOTEL%';
-- update call_type set category_name = 'misc' where type_name like 'MCAB%';
-- update call_type set category_name = 'misc' where type_name like 'MSC%';
-- update call_type set category_name = 'misc' where type_name like 'PERSNL ASST%';
-- update call_type set category_name = 'misc' where type_name like 'P&W%';
-- update call_type set category_name = 'misc' where type_name like 'REPO%';
-- update call_type set category_name = 'misc' where type_name like 'SIG 17%';
-- update call_type set category_name = 'misc' where type_name like 'UNWANTED%';
update call_type set category_name = 'ems' where type_name like '10-52%';
update call_type set category_name = 'ems' where type_name like '10-52 ALLERG%';
update call_type set category_name = 'ems' where type_name like '10-52 BIRTH%';
update call_type set category_name = 'ems' where type_name like '10-52 CP%';
update call_type set category_name = 'ems' where type_name like '10-52 DIABET%';
update call_type set category_name = 'ems' where type_name like '10-52 FALL%';
update call_type set category_name = 'ems' where type_name like '10-52 HEAD%';
update call_type set category_name = 'ems' where type_name like '10-52 HEART%';
update call_type set category_name = 'ems' where type_name like '10-52 LIFT%';
update call_type set category_name = 'ems' where type_name like '10-52 SEIZU%';
update call_type set category_name = 'ems' where type_name like '10-52 STROKE%';
update call_type set category_name = 'ems' where type_name like '10-52 TRANS%';
update call_type set category_name = 'ems' where type_name like '10-52 UNK%';
update call_type set category_name = 'ems' where type_name like 'IND EXPOSURE%';
update call_type set category_name = 'ems' where type_name like 'MEDICAL ALRM%';
update call_type set category_name = 'ems' where type_name like 'OVERDOSE%';
update call_type set category_name = 'misc' where category_name is null;
--update call_type set category_id = call_category.category_id from call_category where call_category.category_name=call_type.category_name;
--UPDATE C SET column1 = A.column1,column2 = B.column2,column3 = A.column1 + B.column2 FROM A JOIN B ON A.id = B.id WHERE C.id = A.id AND (C.column1, C.column2, C.column3) IS DISTINCT FROM (A.column1, B.column2, A.column1 + B.column2);
insert into call_category (category_name) select distinct category_name from call_type;
-- create table call_group (group_id real, group_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- Now take care of response_csv
-- -- agency processing: create the tables, populate the agency table, and from those id's populate the response table
-- create table agency (agency_id serial unique, agency_abbrev varchar, agency_name varchar, city varchar);
insert into agency (agency_abbrev, agency_name) select distinct abbrev,name from response_csv;
-- create table agency_type (agency_id serial unique, type_name char(15));
-- create table response (call_id char(15), agency_id real);
insert into response (call_id, agency_id) select calls_csv.call_id, agency.agency_id from calls_csv inner join response_csv on response_csv.call_id=calls_csv.call_id inner join agency on response_csv.abbrev=agency.agency_abbrev;
update call_type set category_id = call_category.category_id from call_category where call_category.category_name = call_type.category_name;
commit;
begin;
-- -- Create the tables
create table calls_csv (call_id char(15), date date, time time without time zone, complaint varchar, address varchar, city varchar, lat real, long real );
create table response_csv (call_id char(15), abbrev varchar, name varchar);
-- -- Populate them from the raw and uncleaned csv
\copy calls_csv from '/cluster/home/datascience/911calls/raw/2015calls.csv' with delimiter ',' csv header;
\copy response_csv from '/cluster/home/datascience/911calls/raw/departmentsdispatched.csv' with delimiter ',' csv;
-- set to initcap so as to make life easier
update calls_csv set city=initcap(city);
-- -- do something to get the additional geocodes we need: postgis? clean up, geocode
-- -- Update all the things
-- update agencies set agency_type = 'law' where agency_name like '%POLICE%';
-- update agencies set agency_type = 'fire' where agency_name like '%FIRE%';
-- update agencies set agency_type = 'law' where agency_name like '%SHERIFF%';
-- Make cities consistent and correct
update calls_csv set city='Richmond' where city like 'Rich%';
update calls_csv set city='Richmond' where city like '%chmond';
update calls_csv set city='Greens Fork' where city like 'Green%ork';
update calls_csv set city='Hagerstown' where city like 'Hage%';
update calls_csv set city='Boston' where city like 'Bost%';
update calls_csv set city='Connersville' where city like 'Connor%';
update calls_csv set city='Cambridge City' where city like 'Cambr%';
update calls_csv set city='New Paris' where city like 'New Paris%';
update calls_csv set city='Union City' where city like 'Union%';
update calls_csv set city='Randolph County' where city like 'Rando%';
update calls_csv set city='Fayette County' where city like 'Fayette%';
update calls_csv set city='Richmond' where city like '1 S 9th%';
update calls_csv set city='Economy' where city like 'Econ%';
update calls_csv set city='Centerville' where city like 'Cnet';
update calls_csv set city='Richmond' where city='Q';
update calls_csv set city='Richmond' where city='Wtt861';
update calls_csv set city='Farmland Area' where city like 'Farmland%';
update calls_csv set city='Richmond' where address like 'RPD';
update calls_csv set city='Richmond' where address like '18 N 22ND ST' and city like ' %';
-- Some things that are generic enough to update to Wayne County alone
update calls_csv set city='Wayne County' where city='Your City';
update calls_csv set city='Wayne County' where city='Unk';
update calls_csv set city='Wayne County' where city='Pu';
-- Very specific problems
update calls_csv set city='Richmond' where address like '%SW 18TH%';
update calls_csv set address= 'Bertschland, 1 E Church St' where city='1 E Church St';
update calls_csv set city='Cambridge City' where city='1 E Church St';
-- -- Modify the response_csv table
update response_csv set name=initcap(name);
update response_csv set name='Negotiators' where name like 'Negg%';
-- -- if no errors, insert the clean data into the appropriate postgres tables
-- -- spread the data downstream to the derived tables
-- -- Take care of calls_csv
-- want to use property cards for the ID over time
create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/geocodes.csv' with delimiter ',' csv header;
insert into place (addr, lat, long, city) select distinct address, lat, long, city from calls_csv;
-- -- locations are processed, now use that
create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
-- fix this one:
insert into calls (call_id, place_id, date, time) select calls_csv.call_id, place.place_id, calls_csv.date, calls_csv.time from calls_csv inner join place on place.addr = calls_csv.address;
-- debug
create view address_align as select calls.call_id, calls.place_id, place.addr, place.place_id as place from calls inner join place on calls.place_id=place.place_id;
-- -- incident hierarchy
create table call_type (complaint_name varchar, type_name char(15), category_id char(15));
insert into call_type (complaint_name, type_name) select distinct upper(complaint), upper(complaint) from calls_csv;
-- -- clean up the complaints
update call_type set type_name = '10-15' where complaint_name like '10-15%';
update call_type set type_name = '10-16' where complaint_name like '10-16%';
update call_type set type_name = '10-31' where complaint_name like '10-31%';
update call_type set type_name = '10-37' where complaint_name like '10-37%';
update call_type set type_name = '10-50' where complaint_name like '10-50%';
update call_type set type_name = '10-52' where complaint_name like '10-52%';
update call_type set type_name = '10-53' where complaint_name like '10-53%';
update call_type set type_name = '10-58' where complaint_name like '10-58%';
update call_type set type_name = '10-59' where complaint_name like '10-59%';
update call_type set type_name = '10-70' where complaint_name like '10-70%';
update call_type set type_name = '10-97' where complaint_name like '10-97%';
update call_type set type_name = '10-11' where complaint_name like '10-11%';
update call_type set type_name = '10-90' where complaint_name like '10-90%';
create table call_category (category_id serial unique, category_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
create table call_group (group_id real, group_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- Now take care of response_csv
-- -- agency processing: create the tables, populate the agency table, and from those id's populate the response table
create table agency (agency_id serial unique, agency_abbrev varchar, agency_name varchar, city varchar);
insert into agency (agency_abbrev, agency_name) select distinct abbrev,name from response_csv;
create table agency_type (agency_id serial unique, type_name char(15));
create table response (call_id char(15), agency_id real);
insert into response (call_id, agency_id) select calls_csv.call_id, agency.agency_id from calls_csv inner join response_csv on response_csv.call_id=calls_csv.call_id inner join agency on response_csv.abbrev=agency.agency_abbrev;
commit;
begin;
-- -- Create the tables
create table calls_csv (call_id char(15), date date, time time without time zone, complaint varchar, address varchar, city varchar, lat real, long real );
create table response_csv (call_id char(15), abbrev varchar, name varchar);
-- -- Populate them from the raw and uncleaned csv
\copy calls_csv from '/cluster/home/datascience/911calls/raw/2015calls.csv' with delimiter',' csv header;
\copy response_csv from '/cluster/home/datascience/911calls/raw/departmentsdispatched.csv' with delimiter',' csv;
-- set to initcap so as to make life easier
update calls_csv set city=initcap(city);
-- -- do something to get the additional geocodes we need: postgis? clean up, geocode
-- -- Update all the things
-- update agencies set agency_type = 'law' where agency_name like '%POLICE%';
-- update agencies set agency_type = 'fire' where agency_name like '%FIRE%';
-- update agencies set agency_type = 'law' where agency_name like '%SHERIFF%';
-- Make cities consistent and correct
update calls_csv set city= 'Richmond' where city like 'Rich%';
update calls_csv set city= 'Richmond' where city like '%chmond';
update calls_csv set city= 'Greens Fork' where city like 'Green%ork';
update calls_csv set city= 'Hagerstown' where city like 'Hage%';
update calls_csv set city= 'Boston' where city like 'Bost%';
update calls_csv set city= 'Connersville' where city like 'Connor%';
update calls_csv set city= 'Cambridge City' where city like 'Cambr%';
update calls_csv set city= 'New Paris' where city like 'New Paris%';
update calls_csv set city= 'Union City' where city like 'Union%';
update calls_csv set city= 'Randolph County' where city like 'Rando%';
update calls_csv set city= 'Fayette County' where city like 'Fayette%';
update calls_csv set city= 'Richmond' where city like '1 S 9th%';
update calls_csv set city= 'Economy' where city like 'Econ%';
update calls_csv set city= 'Centerville' where city like 'Cnet';
update calls_csv set city= 'Richmond' where city= 'Q';
update calls_csv set city= 'Richmond' where city= 'Wtt861';
update calls_csv set city= 'Farmland Area' where city like 'Farmland%';
update calls_csv set city= 'Richmond' where address like 'RPD';
update calls_csv set city= 'Richmond' where address like '18 N 22ND ST' and city like ' %';
-- Some things that are generic enough to update to Wayne County alone
update calls_csv set city= 'Wayne County' where city= 'Your City';
update calls_csv set city= 'Wayne County' where city= 'Unk';
update calls_csv set city= 'Wayne County' where city= 'Pu';
-- Very specific problems
update calls_csv set city= 'Richmond' where address like '%SW 18TH%';
update calls_csv set address= 'Bertschland, 1 E Church St' where city= '1 E Church St';
update calls_csv set city= 'Cambridge City' where city= '1 E Church St';
-- -- Modify the response_csv table
update response_csv set name=initcap(name);
update response_csv set name= 'Negotiators' where name like 'Negg%';
-- -- if no errors, insert the clean data into the appropriate postgres tables
-- -- spread the data downstream to the derived tables
-- -- Take care of calls_csv
-- want to use property cards for the ID over time
create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/geocodes.csv' with delimiter',' csv header;
insert into place (addr, lat, long, city) select distinct address, lat, long, city from calls_csv;
-- -- locations are processed, now use that
create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
-- fix this one:
insert into calls (call_id, place_id, date, time) select calls_csv.call_id, place.place_id, calls_csv.date, calls_csv.time from calls_csv inner join place on place.addr = calls_csv.address;
-- debug
create view address_align as select calls.call_id, calls.place_id, place.addr, place.place_id as place from calls inner join place on calls.place_id=place.place_id;
-- -- incident hierarchy
create table call_type (complaint_name varchar, type_name char(15), category_name varchar, category_id real);
insert into call_type (complaint_name, type_name) select distinct upper(complaint), upper(complaint) from calls_csv;
-- -- clean up the complaints
update call_type set type_name = '10-15' where complaint_name like '10-15%';
update call_type set type_name = '10-16' where complaint_name like '10-16%';
update call_type set type_name = '10-31' where complaint_name like '10-31%';
update call_type set type_name = '10-37' where complaint_name like '10-37%';
update call_type set type_name = '10-50' where complaint_name like '10-50%';
update call_type set type_name = '10-52' where complaint_name like '10-52%';
update call_type set type_name = '10-53' where complaint_name like '10-53%';
update call_type set type_name = '10-58' where complaint_name like '10-58%';
update call_type set type_name = '10-59' where complaint_name like '10-59%';
update call_type set type_name = '10-70' where complaint_name like '10-70%';
update call_type set type_name = '10-97' where complaint_name like '10-97%';
update call_type set type_name = '10-11' where complaint_name like '10-11%';
update call_type set type_name = '10-90' where complaint_name like '10-90%';
create table call_category (category_id serial unique, category_name varchar);
-- insert into call_category (category_name) select cat names;
-- update call_type set call_type.category_id = call_category.category_id;
-- -- A giant block of redundant-looking code that picks category names
update call_type set category_name = 'law' where type_name like '10-10%';
update call_type set category_name = 'law' where type_name like '10-11%';
update call_type set category_name = 'law' where type_name like '10-14%';
update call_type set category_name = 'law' where type_name like '10-15%';
update call_type set category_name = 'law' where type_name like '10-16%';
update call_type set category_name = 'law' where type_name like '10-31%';
update call_type set category_name = 'law' where type_name like '10-32%';
update call_type set category_name = 'law' where type_name like '10-37%';
update call_type set category_name = 'law' where type_name like '10-38%';
update call_type set category_name = 'law' where type_name like '10-45%';
update call_type set category_name = 'law' where type_name like '10-53%';
update call_type set category_name = 'law' where type_name like '10-55%';
update call_type set category_name = 'law' where type_name like '10-56%';
update call_type set category_name = 'law' where type_name like '10-58%';
update call_type set category_name = 'law' where type_name like '10-59%';
update call_type set category_name = 'law' where type_name like '10-70%';
update call_type set category_name = 'law' where type_name like '10-90%';
update call_type set category_name = 'law' where type_name like '10-94%';
update call_type set category_name = 'law' where type_name like '10-96%';
update call_type set category_name = 'law' where type_name like '10-97%';
update call_type set category_name = 'law' where type_name like 'ARTICLE RECV%';
update call_type set category_name = 'law' where type_name like 'AUDIBLE ALRM%';
update call_type set category_name = 'law' where type_name like 'BAR CHECK%';
update call_type set category_name = 'law' where type_name like 'BIKE PICKUP%';
update call_type set category_name = 'law' where type_name like 'BUSINESS ALM%';
update call_type set category_name = 'law' where type_name like 'BUSINESS CK%';
update call_type set category_name = 'law' where type_name like 'CHILD ABUSE%';
update call_type set category_name = 'law' where type_name like 'CHILD NEGLEC%';
update call_type set category_name = 'law' where type_name like 'DRIVE OFF%';
update call_type set category_name = 'law' where type_name like 'EMER DETENT%';
update call_type set category_name = 'law' where type_name like 'EXTRA ATTEN%';
update call_type set category_name = 'law' where type_name like 'FALSE INFORM%';
update call_type set category_name = 'law' where type_name like 'FLAGGED DOWN%';
update call_type set category_name = 'law' where type_name like 'FOUND ARTCLE%';
update call_type set category_name = 'law' where type_name like 'GRAFFITI%';
update call_type set category_name = 'law' where type_name like 'HARASSMENT%';
update call_type set category_name = 'law' where type_name like 'HOLDUP ALARM%';
update call_type set category_name = 'law' where type_name like 'HOMICIDE%';
update call_type set category_name = 'law' where type_name like 'INCORIG JUV%';
update call_type set category_name = 'law' where type_name like 'INV%';
update call_type set category_name = 'law' where type_name like 'INV BAT ADLT%';
update call_type set category_name = 'law' where type_name like 'INV BAT JUV%';
update call_type set category_name = 'law' where type_name like 'INV B&E BUS%';
update call_type set category_name = 'law' where type_name like 'INV B&E OTH%';
update call_type set category_name = 'law' where type_name like 'INV B&E RES%';
update call_type set category_name = 'law' where type_name like 'INV CRIM REC%';
update call_type set category_name = 'law' where type_name like 'INV FORGERY%';
update call_type set category_name = 'law' where type_name like 'INV FRAUD%';
update call_type set category_name = 'law' where type_name like 'INV INVASION%';
update call_type set category_name = 'law' where type_name like 'INV MISCHIEF%';
update call_type set category_name = 'law' where type_name like 'INV ROBBERY%';
update call_type set category_name = 'law' where type_name like 'INV THEFT%';
update call_type set category_name = 'law' where type_name like 'INV VEH THFT%';
update call_type set category_name = 'law' where type_name like 'JUV PROBLEM%';
update call_type set category_name = 'law' where type_name like 'KIDNAP/ABDUC%';
update call_type set category_name = 'law' where type_name like 'LOCK OUT%';
update call_type set category_name = 'law' where type_name like 'LOST PLATE%';
update call_type set category_name = 'law' where type_name like 'MISSING PER%';
update call_type set category_name = 'law' where type_name like 'OPEN BUILDNG%';
update call_type set category_name = 'law' where type_name like 'OPEN VEHICLE%';
update call_type set category_name = 'law' where type_name like 'PEACE OFFICR%';
update call_type set category_name = 'law' where type_name like 'PROSTITUTION%';
update call_type set category_name = 'law' where type_name like 'PURSE SNATCH%';
update call_type set category_name = 'law' where type_name like 'PURSUIT FOOT%';
update call_type set category_name = 'law' where type_name like 'PURSUIT VEH%';
update call_type set category_name = 'law' where type_name like 'RECKLESS DRV%';
update call_type set category_name = 'law' where type_name like 'RESIDENTL AL%';
update call_type set category_name = 'law' where type_name like 'ROAD RAGE%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY%';
update call_type set category_name = 'law' where type_name like 'RUNAWAY RET%';
update call_type set category_name = 'law' where type_name like 'SEARCH WRRNT%';
update call_type set category_name = 'law' where type_name like 'SEX OFFENDER%';
update call_type set category_name = 'law' where type_name like 'SHOOTING%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER A%';
update call_type set category_name = 'law' where type_name like 'SHOPLIFTER J%';
update call_type set category_name = 'law' where type_name like 'SHOTS FIRED%';
update call_type set category_name = 'law' where type_name like 'SIG 23%';
update call_type set category_name = 'law' where type_name like 'SIG 27%';
update call_type set category_name = 'law' where type_name like 'SIG 60%';
update call_type set category_name = 'law' where type_name like 'SIREN%';
update call_type set category_name = 'law' where type_name like 'SLIDE OFF%';
update call_type set category_name = 'law' where type_name like 'SPEAK W/OFF%';
update call_type set category_name = 'law' where type_name like 'SPECIAL EVNT%';
update call_type set category_name = 'law' where type_name like 'STABBING%';
update call_type set category_name = 'law' where type_name like 'STALKING%';
update call_type set category_name = 'law' where type_name like 'SUICIDAL SUB%';
update call_type set category_name = 'law' where type_name like 'TRAFFIC DTL%';