*********************************************
*CreateWAVoterForAnalysis.do
**
*Create two data files for Washington Vote-By-Mail analysis. 
**
*Take Stata files created by WAVoterToStata.do and
*match registrants across 2008 and 2010 voter files, 
*saving as WaVoterForAnalysis0810.dta. Second, match registrants 
*across 2006 and 2008 voter files, saving as 
*WaVoterForAnalysis0608.dta.
**
*Alan Gerber, Gregory Huber, Seth Hill, "Identifying the Effects of Elections Held All-Mail on
*Turnout: Staggered Reform in the Evergreen State."
*********************************************

clear
capture set mem 5g
capture set mem 8g
set more off
log using CreateWAVoterForAnalysis2.log, text replace
di c(current_date)+ " " + c(current_time)

/*************************************
Create files needed for later merges.
 + 2008 household identifier to block group mapping.
 + Vote histories from 2008 file for the 2004, 2006, & 2008 elections.
 + Vote histories from 2010 file for the 2008 & 2010 election.
 + Extract of registrant variables from 2010 file to make sure same
   statevoterid in 2008 and 2010 matches same individual.
*************************************/

*Vote histories from 2008 file for the 2004, 2006, & 2008 elections.
use WA_Voter_All_Histories_200812.dta, clear
*Keep records from 2004, 2006, and 2008 general elections; fuzzy matching not
*necessary as records are well kept.
keep if voted == "11/02/2004" | voted == "11/07/2006" | voted == "11/04/2008" | voted == "11/08/2005" |voted == "11/06/2007"
*Reshape to wide on statevoterid.
keep statevoterid voted
replace voted = "2004g" if voted == "11/02/2004"
replace voted = "2005g" if voted == "11/08/2005"
replace voted = "2006g" if voted == "11/07/2006"
replace voted = "2007g" if voted == "11/06/2007"
replace voted = "2008g" if voted == "11/04/2008"
rename voted elec
*Drop duplicates
duplicates drop
gen vote = 1
reshape wide vote, i(statevoterid) j(elec) string
replace vote2004g = 0 if vote2004g == .
replace vote2005g = 0 if vote2005g == .
replace vote2006g = 0 if vote2006g == .
replace vote2007g = 0 if vote2007g == .
replace vote2008g = 0 if vote2008g == .
tab vote2004g vote2006g, col miss nokey
tab vote2005g vote2007g, col miss nokey
tab vote2008g vote2006g, col miss nokey
sort statevoterid
isid statevoterid
count
save "temp/WA_GeneralHistories_200812.dta", replace

*Vote histories from 2010 file for the 2010 election.
use WA_Voter_All_Histories_201012.dta, clear
keep if voted == "11/02/2010"  | voted == "11/04/2008"
keep statevoterid voted
*Reshape to wide on statevoterid.
keep statevoterid voted
replace voted = "2008gfrom10file" if voted == "11/04/2008"
replace voted = "2010g" if voted == "11/02/2010"
rename voted elec
*Drop duplicates
duplicates drop
gen vote = 1
reshape wide vote, i(statevoterid) j(elec) string
replace vote2008gfrom10file = 0 if vote2008g == .
replace vote2010g = 0 if vote2010g == .
tab vote2008g vote2010g, col miss nokey
sort statevoterid
isid statevoterid
count
save "temp/WA_2010GeneralHistories.dta", replace

*Extract of registrant variables from 2008 file.
use "WA_Voter_All_Registrants_200812.dta", clear
local keepvars countyvoterid countycode title fname mname lname namesuffix birthdate gender absenteetype
keep statevoterid `keepvars'
foreach v of varlist `keepvars' {
  rename `v' `v'2008
}
compress `keepvars'
*Drop duplicates.
duplicates drop
sort statevoterid
isid statevoterid
save "temp/WA_2008Registrants.dta", replace

*Extract of registrant variables from 2010 file.
use "WA_Voter_All_Registrants_201012.dta", clear
local keepvars countyvoterid countycode title fname mname lname namesuffix birthdate gender absenteetype
keep statevoterid `keepvars'
foreach v of varlist `keepvars' {
  rename `v' `v'2010
}
compress `keepvars'
*Drop duplicates.
duplicates drop
sort statevoterid
isid statevoterid
save "temp/WA_2010Registrants.dta", replace

*************************************
*************************************
*2008 to 2010 matched file.
*************************************
*************************************

use WA_Voter_All_Registrants_200812.dta, clear

*Join vote histories from 2008 voter file.
sort statevoterid
isid statevoterid
joinby statevoterid using "temp/WA_GeneralHistories_200812.dta", unmatched(master) _merge(merge_08hist)
label variable merge_08hist "Merge result for statevoterid to turnout histories from 2008 history file."
tab merge_08hist
replace vote2004g = 0 if vote2004g == .
replace vote2005g = 0 if vote2005g == .
replace vote2006g = 0 if vote2006g == .
replace vote2007g = 0 if vote2007g == .
replace vote2008g = 0 if vote2008g == .
*Field in voter file lastvoted may have more recent turnout record.
replace vote2008g = 1 if lastvoted == mdy(11,4,2008)
replace vote2007g = 1 if lastvoted == mdy(11,6,2007)
replace vote2006g = 1 if lastvoted == mdy(11,7,2006)
replace vote2005g = 1 if lastvoted == mdy(11,8,2005)
replace vote2004g = 1 if lastvoted == mdy(11,2,2004)

*Join 2010 registrant variables to check that statevoterid matches to same person.
sort statevoterid
isid statevoterid
joinby statevoterid using "temp/WA_2010Registrants.dta", unmatched(master) _merge(merge_10reg)
label variable merge_10reg "Merge result for statevoterid to registrant variables from 2010 registrant file."
tab merge_10reg

*Check on merge through registrant variables.
gen check_fname = (fname == fname2010)
gen check_lname = (lname == lname2010)
gen check_bdate = (birthdate == birthdate2010)
gen check_gendr = (gender == gender2010)
egen checker = rowtotal(check_*)
tab checker merge_10reg
*Random sample of failed checks. All look to be name changes or typo fixes.
sort statevoterid
set seed 654321
gen rand = runiform() + (checker==4)
sort rand
bys checker merge_10reg: li checker fname* lname* birthdate* if _n < 20  & merge_10reg == 3
*Check on and drop poor matches; if checker == 0, none of the checks above are equivalent,
*so assuming some screw up in the record keeping.
count if checker < 1 & merge_10reg == 3
li checker fname* lname* birthdate* if checker == 0 & merge_10reg == 3
drop if checker < 1 & merge_10reg == 3
drop rand checker check_* 

*How many moved across county lines?
gen samecounty = (countycode == countycode2010)
tab samecounty if merge_10reg == 3
*Drop matched movers.
drop if samecounty == 0 & merge_10reg == 3
drop samecounty

*Join vote history for 2010 general election from 2010 voter file.
sort statevoterid
isid statevoterid
joinby statevoterid using "temp/WA_2010GeneralHistories.dta", unmatched(master) _merge(merge_10hist)
label variable merge_10hist "Merge result for statevoterid to turnout histories from 2010 history file."
tab merge_10hist
replace vote2010g = 0 if vote2010g == .
tab vote2008g vote2010g, miss col nokey

*Drop individual identifying variables
drop gender2010 countyvoterid* title* fname* mname* lname* namesuffix* regstnum-regzip levycode* mail*

*Make sure precinctcode is string.
gen precinctcode2 = "a"+string(precinctcode)
drop precinctcode
rename precinctcode2 precinctcode

*Save.
qui compress
isid statevoterid
desc
save WaVoterForAnalysis0810.dta, replace

*************************************
*************************************
*2006 to 2008 matched file.
*************************************
*************************************

use WA_Voter_All_Registrants_200701.dta, clear
drop vote2006g

*Join 2008 registrant variables to check that statevoterid matches to same person.
sort statevoterid
isid statevoterid
joinby statevoterid using "temp/WA_2008Registrants.dta", unmatched(master) _merge(merge_08reg)
label variable merge_08reg "Merge result for statevoterid to registrant variables from 2008 registrant file."
tab merge_08reg

*Check on merge through registrant variables.
gen check_fname = (fname == fname2008)
gen check_lname = (lname == lname2008)
*birthdate is string.
gen bdate2 = date(birthdate,"MDY")
format bdate2 %dD_m_Y
drop birthdate
rename bdate2 birthdate
gen check_bdate = (birthdate == birthdate2008)
gen check_gendr = (gender == gender2008)
egen checker = rowtotal(check_*)
tab checker merge_08reg
*Random sample of failed checks. All look to be name changes or typo fixes.
sort statevoterid
set seed 654321
gen rand = runiform() + (checker==4)
sort rand
bys checker merge_08reg: li checker fname* lname* birthdate* if _n < 20  & merge_08reg == 3
*Check on and drop poor matches; if checker == 0, none of the checks above are equivalent,
*so assuming some screw up in the record keeping.
count if checker < 1 & merge_08reg == 3
li checker fname* lname* birthdate* if checker == 0 & merge_08reg == 3
drop if checker < 1 & merge_08reg == 3
drop rand checker check_* 

*How many moved across county lines?
gen samecounty = (countycode == countycode2008)
tab samecounty if merge_08reg == 3
*Drop matched movers.
drop if samecounty == 0 & merge_08reg == 3
drop samecounty

*Join vote histories from 2008 voter file.
sort statevoterid
isid statevoterid
joinby statevoterid using "temp/WA_GeneralHistories_200812.dta", unmatched(master) _merge(merge_08hist)
label variable merge_08hist "Merge result for statevoterid to turnout histories from 2008 history file."
tab merge_08hist
replace vote2004g = 0 if vote2004g == .
replace vote2005g = 0 if vote2005g == .
replace vote2006g = 0 if vote2006g == .
replace vote2007g = 0 if vote2007g == .
replace vote2008g = 0 if vote2008g == .
*Field in 2007 voter file lastvoted may have more recent turnout record.
replace vote2007g = 1 if lastvoted == "11/06/2007"
replace vote2006g = 1 if lastvoted == "11/07/2006"
replace vote2005g = 1 if lastvoted == "11/08/2005"
replace vote2004g = 1 if lastvoted == "11/02/2004"

*********
*Join in 2010 turnout.
*********

*Join 2010 registrant variables to check that statevoterid matches to same person.
sort statevoterid
isid statevoterid
joinby statevoterid using "temp/WA_2010Registrants.dta", unmatched(master) _merge(merge_10reg)
label variable merge_10reg "Merge result for statevoterid to registrant variables from 2010 registrant file."
tab merge_10reg

*Check on merge through registrant variables.
gen check_fname = (fname == fname2010)
gen check_lname = (lname == lname2010)
gen check_bdate = (birthdate == birthdate2010)
gen check_gendr = (gender == gender2010)
egen checker = rowtotal(check_*)
tab checker merge_10reg
*Random sample of failed checks. All look to be name changes or typo fixes.
sort statevoterid
set seed 654321
gen rand = runiform() + (checker==4)
sort rand
bys checker merge_10reg: li checker fname* lname* birthdate* if _n < 20  & merge_10reg == 3
*Check on and note poor matches; if checker == 0, none of the checks above are equivalent,
*so assuming some screw up in the record keeping.
count if checker < 1 & merge_10reg == 3
li checker fname* lname* birthdate* if checker == 0 & merge_10reg == 3
gen bad06to10match = (checker < 1 & merge_10reg == 3)
label variable bad06to10match "Registrant fields from 2006 file do not match to registrant fields from 2010 file."

*How many moved across county lines?
gen samecounty06and10 = (countycode == countycode2010)
label variable samecounty06and10 "Registrant in same county in 2006 and 2010 voter files."
tab samecounty06and10 if merge_10reg == 3

*Join vote history for 2010 general election from 2010 voter file. Also, update 2008 vote
*history if it is one in the 2010 history file.
sort statevoterid
isid statevoterid
joinby statevoterid using "temp/WA_2010GeneralHistories.dta", unmatched(master) _merge(merge_10hist)
label variable merge_10hist "Merge result for statevoterid to turnout histories from 2010 history file."
tab merge_10hist
replace vote2010g = 0 if vote2010g == .
tab vote2008g vote2010g, miss col nokey
replace vote2008g = 1 if vote2008g == 0 & vote2008gfrom10file == 1
tab vote2008g vote2010g, miss col nokey

*Note filters needed for 2010 analysis.
notes: "To analyze 2010 turnout, should filter on variables bad06to10match and samecounty06and10."

*Drop individual identifying variables
drop gender2008 countyvoterid* title* fname* mname* lname* namesuffix* regstnum-regzip levycode* mail*

*Make sure precinctcode is string.
replace precinctcode = "a"+precinctcode

*Save.
qui compress
isid statevoterid
desc

* Delete unnecc variables
drop legislativedistrict congressionaldistrict registrationdate statuscode hh_group num_in_hh

compress
save WaVoterForAnalysis0608.dta, replace

*Remove temp files.
rm "temp/WA_GeneralHistories_200812.dta"
rm "temp/WA_2010GeneralHistories.dta"
rm "temp/WA_2008Registrants.dta"
rm "temp/WA_2010Registrants.dta"
