Access via API and R

An application programming interface (API) is a way for two or more computer programs to communicate with each other.

More information about how to amend API links can be found here. Useful introductions to using APIs in R can be found here.

11.1 Ex. Load the first 25 rows (default) of data

# install.packages(c("httr", "jsonlite"))
library(httr)
library(jsonlite)
library(dplyr)

# link to the API
api_link <- "https://apps-st.fisheries.noaa.gov/ods/foss/afsc_groundfish_survey/"

res <- httr::GET(url = api_link)
# res # Test connection
data <- jsonlite::fromJSON(base::rawToChar(res$content))
# names(data)
tibble::as_tibble(data$items) %>% 
  dplyr::mutate_if(is.character, type.convert, as.is = TRUE) %>%
  dplyr::mutate(across(where(is.numeric), round, 3)) %>%
  head(3) %>%
  flextable::flextable() %>%
  flextable::theme_zebra() %>%
  flextable::colformat_num(x = ., j = c("year", "cruise", "species_code", "tsn", "ak_survey_id"), big.mark = "")

Load the first 25 rows (default) of data.

year	srvy	survey	survey_id	cruise	haul	stratum	station	vessel_name	vessel_id	date_time	latitude_dd	longitude_dd	species_code	common_name	scientific_name	taxon_confidence	cpue_kgha	cpue_kgkm2	cpue_kg1000km2	cpue_noha	cpue_nokm2	cpue_no1000km2	weight_kg	count	bottom_temperature_c	surface_temperature_c	depth_m	distance_fished_km	net_width_m	net_height_m	area_swept_ha	duration_hr	tsn	ak_survey_id	links
2002	AI	Aleutian Islands Bottom Trawl Survey	52	200201	6	722	307-63	Vesteraalen	94	05/17/2002 18:56:58	53.737	-167.016	95020	feathery bryozoan	Eucratea loricata	Low	0.017	1.749	1,749.445				0.044	0	4.1	5.3	187	1.561	16.112	7.25	2.515	0.28	155809	1917453	[[data.frame]]
2002	AI	Aleutian Islands Bottom Trawl Survey	52	200201	6	722	307-63	Vesteraalen	94	05/17/2002 18:56:58	53.737	-167.016	79000	squid unid.	Decapodiformes	High	0.022	2.227	2,226.567	3.181	318.081	318,080.93	0.056	8	4.1	5.3	187	1.561	16.112	7.25	2.515	0.28		1917454	[[data.frame]]
2002	AI	Aleutian Islands Bottom Trawl Survey	52	200201	6	722	307-63	Vesteraalen	94	05/17/2002 18:56:58	53.737	-167.016	24191	shortfin eelpout	Lycodes brevipes	High	0.036	3.578	3,578.410	0.795	79.520	79,520.23	0.090	2	4.1	5.3	187	1.561	16.112	7.25	2.515	0.28	165258	1917455	[[data.frame]]

# install.packages(c("httr", "jsonlite"))
library(httr)
library(jsonlite)
library(dplyr)

# link to the API
api_link <- "https://apps-st.fisheries.noaa.gov/ods/foss/afsc_groundfish_survey/"

res <- httr::GET(url = api_link)
# res # Test connection
data <- jsonlite::fromJSON(base::rawToChar(res$content))
# names(data)
tibble::as_tibble(data$items) %>% 
  dplyr::mutate_if(is.character, type.convert, as.is = TRUE) %>%
  dplyr::mutate(across(where(is.numeric), round, 3)) %>%
  head(3) %>%
  flextable::flextable() %>%
  flextable::theme_zebra() %>%
  flextable::colformat_num(x = ., j = c("year", "cruise", "species_code", "tsn", "ak_survey_id"), big.mark = "")

Load the first 25 rows (default) of data.

year	srvy	survey	survey_id	cruise	haul	stratum	station	vessel_name	vessel_id	date_time	latitude_dd	longitude_dd	species_code	common_name	scientific_name	taxon_confidence	cpue_kgha	cpue_kgkm2	cpue_kg1000km2	cpue_noha	cpue_nokm2	cpue_no1000km2	weight_kg	count	bottom_temperature_c	surface_temperature_c	depth_m	distance_fished_km	net_width_m	net_height_m	area_swept_ha	duration_hr	tsn	ak_survey_id	links
2002	AI	Aleutian Islands Bottom Trawl Survey	52	200201	6	722	307-63	Vesteraalen	94	05/17/2002 18:56:58	53.737	-167.016	95020	feathery bryozoan	Eucratea loricata	Low	0.017	1.749	1,749.445				0.044	0	4.1	5.3	187	1.561	16.112	7.25	2.515	0.28	155809	1917453	[[data.frame]]
2002	AI	Aleutian Islands Bottom Trawl Survey	52	200201	6	722	307-63	Vesteraalen	94	05/17/2002 18:56:58	53.737	-167.016	79000	squid unid.	Decapodiformes	High	0.022	2.227	2,226.567	3.181	318.081	318,080.93	0.056	8	4.1	5.3	187	1.561	16.112	7.25	2.515	0.28		1917454	[[data.frame]]
2002	AI	Aleutian Islands Bottom Trawl Survey	52	200201	6	722	307-63	Vesteraalen	94	05/17/2002 18:56:58	53.737	-167.016	24191	shortfin eelpout	Lycodes brevipes	High	0.036	3.578	3,578.410	0.795	79.520	79,520.23	0.090	2	4.1	5.3	187	1.561	16.112	7.25	2.515	0.28	165258	1917455	[[data.frame]]

11.2 Ex. Load the first 10,000 rows of data

# Not run because too big:
res <- httr::GET(url = paste0(api_link, "?offset=0&limit=10000"))
data <- jsonlite::fromJSON(base::rawToChar(res$content))
print(paste0("rows: ", dim(data$items)[1], "; cols: ", dim(data$items)[2]))

[1] "rows: 10000; cols: 36"

11.3 Ex. Filter by Year

Show all the data greater than the year 2020.

res <- httr::GET(url = paste0(api_link, '?q={"year":{"$gt":2020}}'))
data <- jsonlite::fromJSON(base::rawToChar(res$content))

as_tibble(data$items) %>% 
  mutate_if(is.character, type.convert, as.is = TRUE) %>%
  head(3) %>%
  dplyr::mutate(across(where(is.numeric), round, 3)) %>%
  dplyr::select(year, srvy, stratum, species_code, cpue_kgkm2) %>%
  flextable::flextable() %>%
  flextable::fit_to_width(max_width = 6) %>% 
  flextable::theme_zebra() %>%
  flextable::colformat_num(x = ., j = c("year", "species_code"), big.mark = "") 

Filter by Year.

year	srvy	stratum	species_code	cpue_kgkm2
2022	AI	793	80540	0.361
2022	AI	793	401	0.903
2022	AI	793	20006	1.661

11.4 Ex. Filter by species name

Show all the data where the product name contains pollock Please note that here the word pollock is case sensitive.

The notation for finding a string is to use % around it. Since % is a reserved character in a URL, you have to replace % with %25.

res <- httr::GET(
  url = paste0(api_link, '?q={"common_name":{"$like":"%25pollock%25"}}'))
data <- jsonlite::fromJSON(base::rawToChar(res$content))

as_tibble(data$items) %>% 
  mutate_if(is.character, type.convert, as.is = TRUE) %>%
  head(3) %>%
  dplyr::mutate(across(where(is.numeric), round, 3)) %>%
  dplyr::select(year, srvy, stratum, species_code, cpue_kgkm2) %>%
  flextable::flextable() %>%
  flextable::fit_to_width(max_width = 6) %>% 
  flextable::theme_zebra() %>%
  flextable::colformat_num(x = ., j = c("year", "species_code"), big.mark = "") 

Filter by species name.

year	srvy	stratum	species_code	cpue_kgkm2
2002	AI	722	21740	775.322
2002	AI	722	21740	10,685.806
2002	AI	721	21740	0.640

11.5 Ex. Combination of year and name filters

Show all the data where years > 2020 and the product name contains pollock

res <- httr::GET(
  url = paste0(api_link, 
               '?q={"year":{"$gt":2020},"common_name":{"$like":"%25pollock%25"}}'))
data <- jsonlite::fromJSON(base::rawToChar(res$content))

as_tibble(data$items) %>% 
  mutate_if(is.character, type.convert, as.is = TRUE) %>%
  head(3) %>%
  dplyr::mutate(across(where(is.numeric), round, 3)) %>%
  dplyr::select(year, srvy, stratum, species_code, cpue_kgkm2) %>%
  flextable::flextable() %>%
  flextable::fit_to_width(max_width = 6) %>% 
  flextable::theme_zebra() %>%
  flextable::colformat_num(x = ., j = c("year", "species_code"), big.mark = "") 

Combination of year and name filters.

year	srvy	stratum	species_code	cpue_kgkm2
2022	AI	793	21740	7,853.632
2022	AI	721	21740	7,235.010
2022	AI	722	21740	22,754.334

11.6 Ex. Combination of year, srvy, stratum

Show all the data where year = 1989, srvy = “EBS”, and stratum is not equal to 81

res <- httr::GET(
  url = paste0(api_link, '?q={"year":1989,"srvy":"EBS","stratum":{"$ne":"81"}}'))
data <- jsonlite::fromJSON(base::rawToChar(res$content))

as_tibble(data$items) %>% 
  mutate_if(is.character, type.convert, as.is = TRUE) %>%
  head(3) %>%
  dplyr::mutate(across(where(is.numeric), round, 3)) %>%
  dplyr::select(year, srvy, stratum, species_code, cpue_kgkm2) %>%
  flextable::flextable() %>%
  flextable::fit_to_width(max_width = 6) %>% 
  flextable::theme_zebra() %>%
  flextable::colformat_num(x = ., j = c("year", "species_code"), big.mark = "") 

Combination of year, srvy, stratum.

year	srvy	stratum	species_code	cpue_kgkm2
1989	EBS	10	66548	1.164
1989	EBS	10	69322	1.164
1989	EBS	10	43000	2.353

11.7 Ex. Visualize CPUE data in distribution map

Pacific cod catch-per-unit-effort estimates for NBS in 2021 and map constructed using akgfmaps.

# res <- httr::GET(
#   url = paste0(api_link, "?offset=0&limit=10000"), 
#   query = list(year = 2021, srvy = "EBS", species_code = 30060))
res <- httr::GET(
  url = paste0(api_link, '?q={"year":2021,"srvy":"NBS","species_code":21720}'))
data_catch <- jsonlite::fromJSON(base::rawToChar(res$content))$items %>% 
  dplyr::select(stratum, station, cpue_kgkm2) 

# zero-fill data (imperfectly, but effective for this example)
res <- httr::GET(
  url = paste0(api_link, '?q={"year":2021,"srvy":"NBS"}offset=0&limit=10000'))
data_haul <- jsonlite::fromJSON(base::rawToChar(res$content))$items %>% 
  dplyr::select(stratum, station, latitude_dd, longitude_dd) %>%
  dplyr::mutate(across(where(is.numeric), round, 3)) %>% 
  dplyr::distinct()

data <- dplyr::left_join(data_haul, data_catch) %>% 
  dplyr::mutate(cpue_kgkm2 = ifelse(is.na(cpue_kgkm2), 0, cpue_kgkm2), 
                dplyr::across(dplyr::everything(), as.numeric)) 

flextable::flextable(data[1:3,]) %>% 
  flextable::fit_to_width(max_width = 6) %>% 
  flextable::theme_zebra() 

Visualize CPUE data in distribution map.

stratum	station	latitude_dd	longitude_dd	cpue_kgkm2
81		61.66434	-172.2655	2,895.258
81		62.33740	-173.1702	1,235.545
70		62.03713	-171.6528	0.000

# devtools::install_github("afsc-gap-products/akgfmaps", build_vignettes = TRUE)
library(akgfmaps)

figure <- akgfmaps::make_idw_map(
  CPUE_KGHA = data$cpue_kgkm2, # calculates the same, regardless of units.  
  LATITUDE = data$latitude_dd, 
  LONGITUDE = data$longitude_dd, 
  region = "bs.north", # Predefined EBS area
  set.breaks = "jenks", # Gets Jenks breaks from classint::classIntervals()
  in.crs = "+proj=longlat", # Set input coordinate reference system
  out.crs = "EPSG:3338", # Set output coordinate reference system
  grid.cell = c(20000, 20000))$plot + # 20x20km grid
  ggplot2::guides(fill=guide_legend(title = "Pacific cod\nCPUE (kg/km2)"))

[inverse distance weighted interpolation]
[inverse distance weighted interpolation]

figure

Visualize CPUE data in distribution map.