Month: July 2019

library(tidyverse)
df_all <- safe_load("../../data/processed_data/df_all_2019_07_09.Rda") 

df <- df_all %>%
  mutate(
    OFFICE = ifelse(OFFICE == "COUNCIL AT LARGE", "COUNCIL AT-LARGE", OFFICE)
  ) %>%
  filter(election == "primary" & PARTY == "DEMOCRATIC") %>%
  rename(warddiv=warddiv19, WARD=WARD19, DIV=DIV19) %>%
  group_by(year, OFFICE, CANDIDATE, PARTY, warddiv, WARD, DIV) %>%
  summarise(votes = sum(VOTES))

office_df <- tribble(
  ~office_raw, ~office, ~nwinners,
  "MAYOR", "Mayor", 1,
  "COUNCIL AT-LARGE", "Council At Large", 5,
  "JUDGE OF THE COURT OF COMMON PLEAS", "Common Pleas", 6,
  "CITY COMMISSIONERS", "Commissioner", 2
) %>%
  mutate(office = factor(office, levels=office))

df <- df %>% inner_join(office_df, by = c("OFFICE" = "office_raw"))

##########
## MAPS
##########

# library(leaflet)
# library(leafsync)
library(sf)

divs <- st_read("../../data/gis/2019/Political_Divisions.shp")  %>%
  rename(warddiv=DIVISION_N)wards <- st_read("../../data/gis/2019/Political_Wards.shp") %>%
  mutate(ward = sprintf("%02d", asnum(WARD_NUM)))df <- df %>% 
  group_by(year, warddiv, OFFICE, PARTY) %>%
  mutate(
    div_votes = sum(votes),
    pvote = votes / div_votes
  ) %>% group_by()

df_ward <- df %>% 
  group_by(year, WARD, OFFICE, CANDIDATE, PARTY) %>%
  summarise(votes = sum(votes)) %>% 
  group_by(year, WARD, OFFICE, PARTY) %>%
  mutate(
    div_votes = sum(votes),
    pvote = votes / div_votes
  )


phila_bbox <- st_bbox(divs)

# phila_leaflet <- leaflet() %>% 
#   addProviderTiles(providers$Stamen.TonerLite) %>%
#   setView(
#     lng=mean(phila_bbox[c("xmin", "xmax")]), 
#     lat=mean(phila_bbox[c("ymin", "ymax")]), 
#     zoom = 10
#   )
# 
# add_geom <- function(map, geom, value, pal, text=NA){
#   return(
#     map %>% addPolygons(
#       data = geom, 
#       weight = 0, color = "white", opacity = 1,
#       fillOpacity = 0.8,  smoothFactor = 0.0,
#       popup = text,
#       fillColor = pal(value)
#     )
#   )
# }
# 
# map_candidate <- function(
#   cand_regex,
#   yr, 
#   colname="pvote",
#   pal=NULL,
#   use_divs=FALSE,
#   max_value=1.0
# ){
#   if(use_divs){
#     candidate_df <- divs %>%
#       left_join(
#         df %>% 
#           filter(year == yr, grepl(cand_regex, CANDIDATE)),
#         by=c("warddiv")
#       ) %>%
#       mutate(
#         geo_name = paste0(
#           "Division ", 
#           substr(warddiv, 1, 2), "-", substr(warddiv, 3, 4)
#         )
#       )
#   } else {
#     candidate_df <- wards %>%
#       left_join(
#         df_ward %>% 
#           filter(year == yr, grepl(cand_regex, CANDIDATE)),
#         by=c("ward"="WARD")
#       ) %>%
#       mutate(geo_name = paste("Ward", ward))
#   }
#   
#   candidate_name <- unique(candidate_df$CANDIDATE)
#   
#   if(is.null(pal)) {
#     pal <- colorNumeric("viridis", domain=c(0, max(candidate_df[[colname]])))
#   }
#   
#   text <-  with(
#       candidate_df,
#       paste0(
#         geo_name, "<br>",
#         format_name(CANDIDATE), "<br>", 
#         scales::comma(votes), " vote", ifelse(votes == 1, "", "s"), 
#         " (", round(100*pvote),"%)")
#     )
#   
#   phila_leaflet %>% add_geom(
#     candidate_df$geometry, 
#     pmin(candidate_df[[colname]], max_value), 
#     pal=pal, 
#     text=text
#   ) %>% addControl(format_name(candidate_name), position = "topright")
# }
# 
# map_candidates <- function(cand_regexes, yr, use_divs=FALSE, max_value=NA, ...){
#   all_regex <- paste0("(",cand_regexes,")", collapse = "|")
# 
#   if(is.na(max_value)){
#     if(use_divs) {
#       max_value <- df %>% 
#         filter(year == yr, grepl(all_regex, CANDIDATE)) %>%
#         with(max(pvote))
#     } else {
#       max_value <- df_ward %>% 
#         filter(year == yr, grepl(all_regex, CANDIDATE)) %>%
#         with(max(pvote))
#     }
#   }
# 
#   pal <- colorNumeric("viridis", domain=c(0, max_value))
#   all_maps <- mapply(
#     map_candidate, 
#     cand_regexes, 
#     MoreArgs = list(
#       yr=yr, pal=pal, 
#       use_divs=use_divs, 
#       max_value=max_value
#     ),
#     SIMPLIFY = FALSE
#   )
#   return(all_maps)
# }

map_candidates <- function(cand_regexes, yr, use_divs=FALSE, max_value=NA, ...){
  all_regex <- paste0("(",cand_regexes,")", collapse = "|")

  if(is.na(max_value)){
    if(use_divs) {
      max_value <- df %>%
        filter(year == yr, grepl(all_regex, CANDIDATE)) %>%
        with(max(pvote))
    } else {
      max_value <- df_ward %>%
        filter(year == yr, grepl(all_regex, CANDIDATE)) %>%
        with(max(pvote))
    }
  }

  all_maps <- ggplot(
    wards %>% left_join(
      df_ward %>%
        filter(year==yr, grepl(all_regex, CANDIDATE)),
      by=c("ward"="WARD")
    ) %>%
      mutate(CANDIDATE=format_name(CANDIDATE))
  ) +
     geom_sf(
       aes(fill = 100*pmin(max_value, pvote)),
       color=NA
     ) +
    facet_wrap(~CANDIDATE, ncol = 2) +
    scale_fill_viridis_c(
      "% of Vote"
    ) +
    theme_map_sixtysix() %+replace%
    theme(legend.position = "right")
  return(all_maps)
}

divs_svd <- safe_load("../quick_election_night_maps/divs_svd.Rda") %>%
  mutate(warddiv = paste0(substr(warddiv, 1, 2), substr(warddiv,4,5)))

divs_svd$cat <- forcats::fct_recode(
  divs_svd$cat,
  "Black Voters"="Black non-wealthy",
  "Wealthy Progressives"="Wealthy",
  "White Moderates"="White non-wealthy"
)

cat_colors <- c(light_blue, light_red, light_orange, light_green)
names(cat_colors) <- levels(divs_svd$cat)      

turnout <- safe_load("../voter_db/outputs/turnout_19.rda") %>%
  group_by() %>%
  filter(`Party Code` == "D")

votes_per_office <- df %>% 
  filter(year == 2019) %>%
  filter(!is.na(office)) %>%
  group_by(warddiv, office) %>%
  summarise(votes = sum(votes)) %>%
  group_by() %>%
  left_join(turnout) %>%
  left_join(
    divs_svd %>% as.data.frame() %>% select(warddiv, cat)
  ) %>%
  group_by(cat, office) %>%
  summarise(votes=sum(votes), turnout=sum(turnout)) %>%
  group_by() %>%
  mutate(votes_per_voter = votes / turnout) %>%
  select(cat, votes, turnout, office, votes_per_voter) 

# votes_per_office %>% filter(office == "Mayor") %>%
#   mutate(pturnout = turnout/sum(turnout))
# 
# votes_per_office %>% group_by(office) %>%
#   mutate(pvotes = votes/sum(votes))

ggplot(
  votes_per_office %>% left_join(office_df),
  aes(x=cat, y=votes_per_voter, fill=cat)
) +
  geom_bar(stat="identity") +
  geom_point(aes(y=nwinners), color=NA) +
  geom_text(
    aes(label = sprintf("%0.2f", votes_per_voter)),
    y=0, vjust=-0.4, color="white"
  )+
  facet_wrap(~office, scales = "free_y") +
  scale_fill_manual(NULL, values=cat_colors) +
    theme_sixtysix() %+replace%
  theme(axis.text.x = element_blank()) +
  ylab("Votes / Voter") +
  xlab(NULL) +
  ggtitle("Votes per voter in different neighborhood blocs")

cat_df <- df %>% 
  filter(year == 2019) %>%
  mutate(candidate_name = format_name(CANDIDATE)) %>%
  filter(!is.na(office)) %>%
  left_join(
    divs_svd %>% as.data.frame() %>% select(warddiv, cat)
  ) %>%
  group_by(office, candidate_name, cat) %>%
  summarise(votes = sum(votes)) %>%
  group_by(office, cat) %>%
  mutate(
    pvote = votes/sum(votes),
    cat_rank = rank(desc(votes), ties.method = "first")
  )

winner_df <- cat_df %>%
  group_by(office, candidate_name) %>%
  summarise(votes = sum(votes)) %>%
  group_by(office) %>%
  mutate(
    pvote = votes/sum(votes),
    office_rank = rank(desc(votes))
  ) %>%
  left_join(office_df) %>%
  mutate(is_winner = office_rank <= nwinners)

win_lines <- winner_df %>% 
  filter(office_rank %in% c(nwinners, nwinners+1)) %>%
  group_by(office) %>%
  summarise(win_line = mean(pvote))

slopes <- votes_per_office %>%
  select(cat, office, votes) %>%
  spread(key=cat, value=votes) %>%
  mutate(slope=-`Wealthy Progressives`/`Black Voters`) %>%
  left_join(win_lines) %>%
  mutate(
    intercept = win_line * (`Wealthy Progressives`+`Black Voters`) / `Black Voters`
  )

scatter_df <- cat_df %>%
  filter(candidate_name != "Write In") %>%
  select(-votes) %>%
  rename(rank=cat_rank) %>%
  gather("key", "value", pvote, rank) %>%
  unite("key", cat, key) %>%
  spread(key, value) %>%
  mutate(last_name = format_name(gsub("^.*\\b(\\S+)$", "\\1", candidate_name)))

plot_scatter <- function(use_office){
  max_break <- scatter_df %>%
    filter(office==use_office) %>%
    with(max(c(`Wealthy Progressives_pvote`, `Black Voters_pvote`)))
  
  breaks <- seq(0,100,by=ifelse(max_break > 0.2, 10, 5))
  
  ggplot(
    scatter_df %>% filter(office==use_office),
    aes(
      x = 100 * `Wealthy Progressives_pvote`,
      y = 100 * `Black Voters_pvote`
    )
  ) +
    geom_text(
      aes(label = last_name)
    ) +
    geom_abline(
      data=slopes  %>% filter(office==use_office), 
      aes(intercept=100*intercept, slope=slope),
      color="grey50",
      linetype="dashed"
    ) +
    # facet_wrap(~office, scales = "free") +
    coord_fixed() +
    scale_x_continuous(
      "Percent of vote among Wealthy Progressive divisions",
      expand=expand_scale(mult=c(0.05, 0.2)),
      breaks=breaks
    ) +
    scale_y_continuous(
      "Percent of vote among Black Voter divisions",
      breaks=breaks
    ) +
    expand_limits(x=0,y=0) +
    theme_sixtysix() +
    ggtitle(sprintf("Voting Bloc results for %s", use_office))
} 

plot_scatter("Commissioner")

plot_scatter("Council At Large")

plot_scatter("Common Pleas")

plot_office <- "Council At Large"

# cat_df %>%
#   filter(office == plot_office) %>%
#   group_by(cat) %>%
#   summarise(
#     pvote_one = sum(pvote * (cat_rank == 1)),
#     pvote_winners = sum(pvote * (cat_rank > 1 & cat_rank <= 5)),
#     pvote_losers = sum(pvote * (cat_rank > 5))
#   )


get_office_plot <- function(
  plot_office, df=cat_df, title="Vote Concentration for %s", show_appendix_text=TRUE
){
  ggplot(
    df %>% filter(office == plot_office),
    aes(x=cat_rank, y=pvote*100, fill=cat)
  ) +
    geom_bar(stat="identity") +
    facet_wrap(~cat) +
    scale_fill_manual(NULL, values=cat_colors, guide=FALSE) +
    geom_vline(
      xintercept=0.5 + office_df$nwinners[office_df$office==plot_office], 
      linetype="dashed"
    ) +
    theme_sixtysix() %+replace%
    theme(axis.text.x = element_blank()) +
    xlab(
      paste0(
        "Candidates sorted by within-cluster rank",
        ifelse(show_appendix_text, "\n(See appendix for names and counts.)", "")
      )
    ) +
    ylab("Percent of Vote") +
    ggtitle(
      sprintf(title, plot_office)
    )
}
get_office_plot("Council At Large")

# cat_df %>%
#   filter(office == "Commissioner") %>%
#   group_by(cat) %>%
#   summarise(
#     pvote_winners = sum(pvote * (cat_rank <= 2)),
#     pvote_losers = sum(pvote * (cat_rank > 2))
#   )

get_office_plot("Commissioner")

# cat_df %>%
#   filter(office == "Common Pleas") %>%
#   group_by(cat) %>%
#   summarise(
#     pvote_winners = sum(pvote * (cat_rank <= 6)),
#     pvote_losers = sum(pvote * (cat_rank > 6))
#   )

get_office_plot("Common Pleas")

cat_df_15 <- df %>% 
  group_by() %>%
  filter(year == 2015) %>%
  mutate(candidate_name = format_name(CANDIDATE)) %>%
  filter(!is.na(office)) %>%
  left_join(
    divs_svd %>% as.data.frame() %>% select(warddiv, cat)
  ) %>%
  group_by(office, candidate_name, cat) %>%
  summarise(votes = sum(votes)) %>%
  group_by(office, cat) %>%
  mutate(
    pvote = votes/sum(votes),
    cat_rank = rank(desc(votes), ties.method = "first")
  )

# cat_df_15 %>%
#   filter(office == "Council At Large") %>%
#   group_by(cat) %>%
#   summarise(
#     pvote_winner = sum(pvote * (cat_rank == 1)),
#     pvote_winners = sum(pvote * (cat_rank <= 5 & cat_rank > 1)),
#     pvote_losers = sum(pvote * (cat_rank > 5))
#   )
# 
# cat_df_15 %>%
#   filter(office == "Commissioner") %>%
#   group_by(cat) %>%
#   summarise(
#     pvote_winners = sum(pvote * (cat_rank <= 2)),
#     pvote_losers = sum(pvote * (cat_rank > 2))
#   )
# 
# get_office_plot(
#   "Council At Large", cat_df_15, title="Vote Concentration for %s, 2015",
#   show_appendix_text=FALSE
# )
# 
# get_office_plot(
#   "Commissioner", cat_df_15, title="Vote Concentration for %s, 2015",
#   show_appendix_text=FALSE
# )

format_columns <- function(table, cols, ...){
  for(col in cols){
    table <- table %>% 
      kableExtra::column_spec(column=col, ...)
  }
  return(table)
}
library(knitr)
library(kableExtra)
get_table_for_office <- function(plot_office){
  overall_df <- cat_df %>%
    group_by() %>%
    filter(office == plot_office) %>% 
    group_by(office, candidate_name) %>% 
    summarise(votes = sum(votes)) %>% 
    group_by() %>% 
    mutate(
      cat="Overall", 
      pvote = votes/sum(votes), 
      cat_rank=rank(desc(votes), ties.method = "first")
    )
  
  table <- kable(
    cat_df %>% 
      group_by() %>%
      filter(office == plot_office) %>%
      bind_rows(overall_df) %>%
      mutate(
        cat=factor(cat, levels=c("Overall", names(cat_colors))),
        votes=paste0(scales::comma(votes), " (", round(100 * pvote),"%)")
      ) %>%
      select(candidate_name, cat, votes, cat_rank) %>%
      rename(
        "Rank in Cluster" = cat_rank,
        "Candidate" = candidate_name
      ) %>%
      gather("key", "value", Candidate, votes) %>%
      mutate(key = format_name(key), blank="     ") %>%
      unite(key, cat, key, sep=": ") %>%
      spread(key, value) %>%
      select(
        "Rank in Cluster", 
        blank,
        starts_with("Overall"),
        starts_with("Black"), 
        starts_with("Wealthy"), 
        starts_with("White"), 
        starts_with("Hispanic")
      )
    ,
    caption=sprintf("Cluster Results for %s", plot_office),
    align='l',
    col.names=c("Rank","", rep(c("Candidate", "Votes"), times=5)),
    padding = "4in"
  ) %>%
    column_spec(2, width = "1em") %>% 
    add_header_above(
      c(" " = 2, "Overall" = 2, "Black Voters" = 2, "Wealthy Progressives" = 2,
        "White Moderates" = 2, "Hispanic North Philly" = 2)
    ) %>%
    kable_styling() %>%
    scroll_box(width="100%")
  
  return(table)
}

get_table_for_office("Commissioner")

get_table_for_office("Council At Large")

get_table_for_office("Common Pleas")