chords.qmd

---
title: "chords"
format: html
editor: visual
# cache: True
---

```{r include=F}
# devtools::install_github("r-music/chorrrds")
require(chorrrds)
require(tidyverse)
require(rvest)
require(ggridges)
require(tictoc)
require(tidytext)
require(forcats)
require(caTools)
require(class)
require(caret)
require(knitr)
source("utils.R")
```

```{r}
top_genres <- c("rock", "gospelreligioso", "sertanejo", "mpb", "alternativo",
             "axe", "funk", "pop", "forro", "samba",
             "bossa-nova", "blues", "jazz", "k-pop", "pagode")
```

```{r include=F}
top_songs <- top_genres %>%
  purrr::map(~{
    .x %>% 
      paste0("https://m.cifraclub.com.br/top-musicas/", .) %>% 
      read_html() %>%
      html_nodes("a") %>%
      html_attr("href") %>%
      .[11:110] %>%
      cbind.data.frame(url = .,
                       genre = .x,
                       ranking = 1:100)
  }) %>%
  purrr::reduce(rbind) %>%
  mutate(genre = gsub("gospelreligioso", "gospel", genre))

top_songs$artist <- sub("^/(.*?)/.*$", "\\1", top_songs$url) %>%
  gsub("-", " ", .) %>%
  gsub("(^|\\s)([a-z])", "\\1\\U\\2", ., perl=T)

top_songs$song <- sub("^/.*?/(.*)/$", "\\1", top_songs$url) %>%
  gsub("-", " ", .) %>%
  gsub("(^|\\s)([a-z])", "\\1\\U\\2", ., perl=T)
```

```{r include=F}
# esse chunk demora cerca de 5 minutos pra rodar
tic()
chords <- read_chords(top_songs)
toc()
```

```{r include=F}
# esse chunk demora cerca de 2 minutos pra rodar
tic()
chords <- chords %>%
  filter(!is.na(artist)) %>%
  transpose_capo(.) %>%
  transpose_chords(.)
toc()
```

```{r include=F}
chordsC <- chords %>%
  mutate(chord = chordC) %>%
  mutate(key = keyC) %>%
  select(-c(chordC, keyC))
```

<br>

# Validação da qualidade do scraping

#### Quantidade de músicas por gênero

```{r echo=F}
aux <- chords %>%
  select(song, artist) %>% 
  unique()

left_join(aux, top_songs, by=c("song", "artist")) %>%
  group_by(genre) %>%
  na.omit() %>%
  unique() %>%
  count(genre) %>%
  arrange(n, genre)
```

#### Quantidade de músicas repetidas

```{r}
top_songs %>% filter(duplicated(top_songs$url)) %>% nrow()
```


<br>

# Análise exploratória inicial

#### Top 20 músicas com mais acordes distintos

```{r echo=F}
chords %>% 
  group_by(song, chord) %>% 
  summarise(distintos = n_distinct(chord)) %>% 
  summarise(cont = n()) %>% 
  mutate(song = fct_reorder(song, cont)) %>% 
  top_n(n = 20) %>% 
  ggplot(aes(y = cont, x = song)) +
  geom_bar(colour = 'white', fill = 'dodgerblue4',
           size = 0.5, alpha = 0.9, stat = "identity") +
  labs(x = '', y = '') +
  geom_text(aes(label = cont), hjust = 1.5, size = 3, colour = 'white') +
  coord_flip() +
  theme_minimal()
```

<br>

#### Densidade de características de acordes

```{r echo=F}
feat_chords <- chords %>%
  select(chord, song) %>% 
  feature_extraction() %>% 
  select(-chord) %>% 
  group_by(song) %>% 
  summarise_all(mean)

dt <- feat_chords %>% 
  gather(group, vars, minor, seventh, 
         seventh_M, sixth, fifth_dim, fifth_aug, 
         fourth, ninth, bass, dimi, augm)

dt$group <- forcats::lvls_revalue(
  dt$group,
  c("Augmented", "Bass", "Diminished",
    "Augm. Fifth", "Dimi. Fifth",
    "Fourth", "Minor", "Ninth", "Seventh",
    "Major Seventh", "Sixth"))

dt %>% 
  ggplot(aes(vars, group, fill = group)) +
  geom_density_ridges(alpha = 0.6) +
  scale_fill_cyclical(values = c("dodgerblue4", "darksalmon")) +
  guides(fill = FALSE) +
  xlim(0, 1) +
  labs(x = "", y = "") +
  theme_minimal()
```

<br>

#### Quantidade média de acordes distintos por gênero

```{r echo=F}
# mgest

aux <- chords %>% 
  group_by(song, chord) %>% 
  summarise(distintos = n_distinct(chord)) %>% 
  summarise(cont = n()) %>% 
  mutate(song = fct_reorder(song, cont))

left_join(aux, top_songs, by="song") %>%
  group_by(genre) %>%
  na.omit() %>%
  summarise(cont = round(mean(cont), 1)) %>%
  ggplot(aes(y = cont, reorder(genre, -cont, decreasing = TRUE))) +
  geom_bar(colour = 'dodgerblue4', fill = 'dodgerblue4', size = 0.5, alpha = 0.9, stat = "identity") +
  geom_text(aes(label = cont), size = 3.5, hjust = 1.2, colour = 'white') +
  labs(x = '', y = '') +
  coord_flip() +
  theme_minimal() +
  theme(axis.text.y = element_text(size = 12),
        axis.text.x = element_text(size = 12),
        plot.title = element_text(size = 16, hjust = 0.5),
        legend.position = "none")
```

<br>

#### Top 5 acordes mais frequentes por gênero

```{r echo=F}
aux <- chords %>%
  group_by(song) %>% 
  count(chord)

left_join(aux, top_songs, by="song") %>%
  group_by(genre) %>%
  na.omit() %>%
  count(chord) %>%
  mutate(prop = scales::percent(n/sum(n))) %>%
  top_n(n, n = 5) %>%
  arrange(genre, desc(n))
```

<br>

#### Top 2 transições mais frequentes por gênero

```{r echo=F}
aux <- chords %>%
  split(.$song) %>% 
  map(chords_ngram, n = 2) %>% 
  bind_rows() %>% 
  group_by(song)

left_join(aux, top_songs, by="song") %>% 
  group_by(genre) %>%
  na.omit() %>%
  count(chords_ngram, key) %>%
  mutate(prop = scales::percent(n/sum(n), 2)) %>%
  top_n(n, n = 2) %>%
  arrange(genre, desc(n))
```

<br>

#### Top 2 progressões mais frequentes por gênero

```{r echo=F}
aux <- chords %>%
  split(.$song) %>% 
  map(chords_ngram, n = 4) %>% 
  bind_rows() %>% 
  group_by(song)

left_join(aux, top_songs, by="song") %>% 
  group_by(genre) %>%
  na.omit() %>%
  count(chords_ngram) %>%
  mutate(prop = scales::percent(n/sum(n), 2)) %>%
  top_n(n, n = 3) %>%
  arrange(genre, desc(n))
```

<br>

#### Top 2 tonalidades mais frequentes por gênero

```{r echo=F}
aux <- chords %>%
  group_by(song) %>%
  count(key)

left_join(aux, top_songs, by="song") %>% 
  group_by(genre) %>%
  na.omit() %>%
  count(key) %>% 
  mutate(prop = scales::percent(n/sum(n), 2)) %>%
  top_n(n, n = 2) %>%
  arrange(genre, desc(n))
```

<br>

#### Proporção de acordes maiores e menores por gênero

```{r echo=F}
aux <- chords %>%
  select(chord, song) %>% 
  feature_extraction() %>% 
  select(-chord) %>% 
  group_by(song) %>% 
  summarise_all(mean)

table_data <- left_join(aux, top_songs, by = "song") %>% 
  group_by(genre) %>%
  rename(gênero = genre) %>%
  na.omit() %>%
  summarise(menores = round(mean(minor), 2)) %>%
  mutate(maiores = 1 - menores) %>%
  arrange(desc(menores)) %>%
  t()

kable(table_data, format = "markdown")
```

<br>

# Análise exploratória após padronização de tom

#### Top 5 acordes mais frequentes por gênero

```{r echo=F}
aux <- chords %>%
  group_by(song) %>% 
  count(chordC)

left_join(aux, top_songs, by="song") %>%
  group_by(genre) %>%
  na.omit() %>%
  count(chordC) %>%
  mutate(prop = scales::percent(n/sum(n))) %>%
  top_n(n, n = 5)
```

<br>

#### Top 2 transições mais frequentes por gênero

```{r echo=F}
aux <- chordsC %>%
  split(.$song) %>% 
  map(chords_ngram, n = 2) %>% 
  bind_rows() %>% 
  group_by(song)

left_join(aux, top_songs, by="song") %>% 
  group_by(genre) %>%
  na.omit() %>%
  count(chords_ngram) %>%
  mutate(prop = scales::percent(n/sum(n), 2)) %>%
  top_n(n, n = 2) %>%
  arrange(genre, desc(n))
```

<br>

#### Top 2 progressões mais frequentes por gênero

```{r echo=F}
aux <- chordsC %>%
  split(.$song) %>% 
  map(chords_ngram, n = 4) %>% 
  bind_rows() %>% 
  group_by(song) %>%
  filter(str_count(chords_ngram, " ") == 3)

left_join(aux, top_songs, by="song") %>% 
  group_by(genre) %>%
  na.omit() %>%
  count(chords_ngram) %>%
  mutate(prop = scales::percent(n/sum(n), 2)) %>%
  top_n(n, n = 2) %>%
  arrange(genre, desc(n))
```

<br>

# TF-IDF

```{r echo=F}
aux <- chordsC %>%
  split(.$song) %>% 
  map(chords_ngram, n = 4) %>% 
  bind_rows() %>% 
  group_by(song) %>%
  filter(str_count(chords_ngram, " ") == 3) %>%
  mutate(chords_ngram = gsub(" ", "-", chords_ngram))

prep_genre <- left_join(aux, top_songs, by="song") %>% 
  group_by(genre) %>%
  select(chords_ngram) %>%
  count(genre, chords_ngram) %>%
  arrange(genre, desc(n))

prep_total <- prep_genre %>% 
  group_by(genre) %>%
  summarize(total = sum(n))

tfidf_genre <- left_join(prep_genre, prep_total) %>%
  bind_tf_idf(chords_ngram, genre, n) %>%
  arrange(desc(tf_idf))

tfidf_genre
```

```{r}
tfidf_genre %>%
  group_by(genre) %>%
  slice_max(tf_idf, n = 3) %>%
  ungroup() %>%
  ggplot(aes(tf_idf, fct_reorder(chords_ngram, tf_idf))) +
  geom_col(show.legend = FALSE, fill = 'dodgerblue4', colour = 'white') +
  facet_wrap(~genre, ncol = 3, scales = "free_y") +
  labs(x = NULL, y = NULL) +
  theme_minimal()
```

<br>