tables populate automatically without errors, script to generate a clean...

tables populate automatically without errors, script to generate a clean geocodes file, aesthetic improvements

911_full_cycle.sql

+\i /cluster/home/cjearley13/DataScience/emergency-calls/911_drop.sql 
+\i /cluster/home/cjearley13/DataScience/emergency-calls/911_tables.sql 
+\i /cluster/home/cjearley13/DataScience/emergency-calls/911_populate.sql 

911_populate.sql

@@ -2,8 +2,14 @@
 
 begin;
 
+--15-00016033,2015-02-27,7:48:59,BUSINESS CK,10000 BLOCK OF WILLIAMSON RD,WILLIAMSBURG,,
+--15-00028812,2015-04-12,9:29:48,BUSINESS CK,10000 BLOCK OF WILLIAMSON RD,WILLIAMSBURG,,
+--15-00037121,2015-05-10,9:16:52,BUSINESS CK,10000 BLOCK OF WILLIAMSON RD,WILLIAMSBURG,,
+--15-00009972,2015-02-05,7:55:16,BUSINESS CK,10000 BLOCK OF WILLIAMSON RD,WILLIAMSBURG,,
+--15-00069858,2015-09-05,21:13:44,TRAFFIC STOP,1000 BLK E MAIN ST,RICHMOND,39.829063,-84.889087
+
 -- -- Populate them from the raw and uncleaned csv
-\copy calls_csv from '/cluster/home/datascience/911calls/raw/2015calls.csv' with delimiter',' csv header;
+\copy calls_csv from '/cluster/home/cjearley13/DataScience/emergency-calls/2015calls.csv' with delimiter ',' csv header;
 \copy response_csv from '/cluster/home/datascience/911calls/raw/departmentsdispatched.csv' with delimiter',' csv;
 
 -- set to initcap so as to make life easier
@@ -54,13 +60,12 @@ update response_csv set name= 'Negotiators' where name like 'Negg%';
 -- Take care of calls_csv
 -- want to use property cards for the ID over time
 -- create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
-\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/geocodes.csv' with delimiter',' csv header;
+\copy place (lat, long, addr, city, state) from '/cluster/home/cjearley13/DataScience/emergency-calls/new.csv' with delimiter',' csv header;
 insert into place (addr, lat, long, city) select distinct address, lat, long, city from calls_csv;
-
 -- -- locations are processed, now use that
--- create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
--- fix this one:
-insert into calls (call_id, place_id, date, time, type_name) select calls_csv.call_id, place.place_id, calls_csv.date, calls_csv.time, calls_csv.complaint from calls_csv inner join place on place.addr = calls_csv.address;
+
+insert into calls (call_id, place_id, date, time, type_name)
+select distinct on (calls_csv.call_id) RIGHT(calls_csv.call_id,5)::integer, place.place_id, calls_csv.date, calls_csv.time, calls_csv.complaint from calls_csv inner join place on place.addr = calls_csv.address;
 
 -- -- incident hierarchy
 insert into call_type (complaint_name, type_name) select distinct upper(complaint), upper(complaint) from calls_csv;

911_tables.sql

 begin;
 
 -- -- Create the tables
-create table calls_csv (call_id char(15), date date, time time without time zone, complaint varchar, address varchar, city varchar, lat real, long real );
-create table response_csv (call_id char(15), abbrev varchar, name varchar);
+create table calls_csv (call_id char(12) primary key, 
+date date, time time without time zone, complaint varchar, 
+address varchar, city varchar, lat real, long real );
 
--- -- Take care of calls_csv
-create table calls (call_id char(15), place_id real, date date, time time, type_name char(15));
-create table place (place_id serial unique, addr varchar, lat real, long real, city varchar, state varchar);
+create table response_csv (call_id char(12), abbrev varchar, name varchar);
+
+create table calls (call_id int primary key, 
+place_id real, date date, time time, type_name char(12));
+
+create table place (place_id serial primary key, 
+addr varchar, lat real, long real, city varchar, state varchar);
 
 -- -- incident hierarchy
-create table call_type (complaint_name varchar, type_name char(15), category_id real);
+create table call_type (complaint_name varchar, type_name char(12), category_id real);
 
 create table call_group (group_id serial unique, group_name varchar);
 insert into call_group (group_name) values ('ems');
@@ -36,7 +41,7 @@ create table agency (agency_id serial unique, agency_abbrev varchar, agency_name
 create table agency_type (agency_id serial unique, type_name char(15));
 
 -- -- agencies with their calls
-create table response (call_id char(15), agency_id real);
+create table response (call_id char(12), agency_id real);
 
 -- -- create any useful views
 

cleangeos.sh

+#!/bin/bash
+
+grep '^\-[0-9]' geocodes.csv >> new.csv
+grep '^\[0-9]' geocodes.csv >> new.csv