How to make an annotation layer relative to a geom_point layer

Question

I would like the annotated text on this chart to be relative to the data points on the line chart because this report is parameterized so the points change with each iteration. How do I make the annotation layer relative to the geom_point layer? This is what I've tried...

degree_line <- degree_summary %>%
  ggplot(mapping = aes(x = chrt_grad, y = proportion, group = 1)) +
  geom_line(size = 1) +
  geom_point(shape = 1, stroke = 1.5) +
  scale_y_continuous(expand = expansion(mult = c(0, .1)),
                     labels = scales::percent,
                     limits = c(0, .4)) +
  annotate(geom = "text", x = 2016.8, y = degree_summary$proportion + .05, label = "4-year rate", 
           size = 3, hjust = 0, color = "#696969") +
  annotate(geom = "text", x = 2015.1, y = degree_summary$proportion + .05, label = "6-year rate", 
           size = 3, hjust = 1, color = "#696969") +
  labs(x = NULL,
       y = NULL)

But this is what I get...

dput(degree_summary)

structure(list(chrt_grad = 2014:2017, graduated = c("Y", "Y", 
"Y", "Y"), school = c("Bonita Vista Senior High", "Bonita Vista Senior High", 
"Bonita Vista Senior High", "Bonita Vista Senior High"), total = c(132L, 
122L, 117L, 92L), proportion = c(0.0172639288516872, 0.0167859108420473, 
0.0187259923175416, 0.0199522880069399)), class = c("grouped_df", 
"tbl_df", "tbl", "data.frame"), row.names = c(NA, -4L), groups = structure(list(
    chrt_grad = 2014:2017, graduated = c("Y", "Y", "Y", "Y"), 
    .rows = structure(list(1L, 2L, 3L, 4L), ptype = integer(0), class = c("vctrs_list_of", 
    "vctrs_vctr", "list"))), row.names = c(NA, -4L), class = c("tbl_df", 
"tbl", "data.frame"), .drop = TRUE))