[PYTHON] Visualize 2ch threads with WordCloud-Scraping-

#Library import
import requests, bs4
import re
import time
import pandas as pd
from urllib.parse import urljoin

#Install fonts locally in Colab
from google.colab import drive
drive.mount("/content/gdrive")
#Create a folder called font at the top of My Drive in your Google Drive in advance, and put the desired font file in it.
#Copy each folder locally to Colab
!cp -a "gdrive/My Drive/font/" "/usr/share/fonts/"

# ------------------------------------------------------------------------
#Preparation
log_database = []  #A list that stores thread information
base_url = "https://www.logsoku.com/search?q=FFRK&p="

#Implementation of web scraping
for i in range(1,4):  #Which page to go back to (here, tentatively up to the 4th page)
  logs_url = base_url+str(i)

  #Scraping processing body
  res = requests.get(logs_url)
  soup = bs4.BeautifulSoup(res.text, "html.parser")

  #What to do when no search results are found
  if soup.find(class_="search_not_found"):break

  #Get table / row where thread information is stored
  thread_table = soup.find(id="search_result_threads")
  thread_rows = thread_table.find_all("tr")

  #Processing for each row
  for thread_row in thread_rows:
    tmp_dict = {}
    tags = thread_row.find_all(class_=["thread","date","length"])

    #Organize the contents
    for tag in tags:
      if "thread" in str(tag):
        tmp_dict["title"] = tag.get("title")
        tmp_dict["link"] = tag.get("href")
      elif "date" in str(tag):
        tmp_dict["date"] = tag.text
      elif "length" in str(tag):
        tmp_dict["length"] = tag.text

    #Only those with more than 50 lesss will be added to the database
    if tmp_dict["length"].isdecimal() and int(tmp_dict["length"]) > 50:
      log_database.append(tmp_dict)

  time.sleep(1)

#Convert to DataFrame
thread_df = pd.DataFrame(log_database)

# ------------------------------------------------------------------------
#Get less from past logs
log_url_base = "http://nozomi.2ch.sc/test/read.cgi/"
res_database = []

for thread in log_database:
  #Board name and bulletin board No. from the past log list.And generate the URL of the past log
  board_and_code_match = re.search("[a-zA-Z0-9_]*?/[0-9]*?/$",thread["link"])
  board_and_code = board_and_code_match.group()
  thread_url = urljoin(log_url_base, board_and_code)

  #HTML extraction from past log page
  res = requests.get(thread_url)
  soup = bs4.BeautifulSoup(res.text, "html5lib")

  tmp_dict = {}
  #Information such as date in the dt tag
  #The comment is stored in the dd tag
  dddt = soup.find_all(["dd","dt"])

  for tag in dddt[::-1]:  #Extract from behind

    #Extract only the date from the dt tag
    if "<dt>" in str(tag):
      date_result = re.search(r"\d*/\d*/\d*",tag.text)  #  "(←'"'I don't care (to avoid display abnormalities of qiita)
      if date_result:
        date_str = date_result.group()
        tmp_dict["date"] = date_str

    #Extract less content from dd tag
    if "<dd>" in str(tag):
      tmp_dict["comment"] = re.sub("\n","",tag.text)

    # tmp_The contents stored in dict are res_Post to database
    if "date" in tmp_dict and "comment" in tmp_dict:
      tmp_dict["thread_title"] = thread["title"]
      res_database.append(tmp_dict)
      tmp_dict = {}

  time.sleep(1)  #promise

#Convert to DataFrame
res_df = pd.DataFrame(res_database)

# ------------------------------------------------------------------------

#Morphological analysis library MeCab and dictionary(mecab-ipadic-NEologd)Installation of
!apt-get -q -y install sudo file mecab libmecab-dev mecab-ipadic-utf8 git curl python-mecab > /dev/null
!git clone --depth 1 https://github.com/neologd/mecab-ipadic-neologd.git > /dev/null 
!echo yes | mecab-ipadic-neologd/bin/install-mecab-ipadic-neologd -n > /dev/null 2>&1
!pip install mecab-python3 > /dev/null

#Error avoidance by symbolic links
!ln -s /etc/mecabrc /usr/local/etc/mecabrc

#Wordcloud installation
!pip install wordcloud

#Combine all the less
sentences = ",".join(res_df["comment"])
sentences_sep = []
n = 10000
for i in range(0,len(sentences), n):
  sentences_sep.append(sentences[i:i + n])

#Les n(=1000)Separate by less and combine with commas
#The purpose of partitioning is because later mecab cannot handle too many characters.
sentences_sep = []
n = 1000
for i in range(0, len(res_df["comment"]), n):
  sentences_sep.append(",".join(res_df["comment"][i: i + n]))

# ------------------------------------------------------------------------
import MeCab

# mecab-ipadic-Specify the path where the neologd dictionary is stored
path = "-d /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd"
#Above path (/usr/~) Can be obtained with the following command
# !echo `mecab-config --dicdir`"/mecab-ipadic-neologd"

#Creating a Tagger object
mecab = MeCab.Tagger(path)

#Perform morphological analysis for each separated group
chasen_list = [mecab.parse(sentence) for sentence in sentences_sep]

word_list = []

# chasen_Decompose list to one line
# ex.Iron giant noun,Proper noun,General,*,*,*,Iron giant,Tetsukyojin,Tetsukyojin）
for chasen in chasen_list:
  for line in chasen.splitlines():
    
    if len(line) <= 1: break

    speech = line.split()[-1]
    if "noun" in speech:
      if  (not "Non-independent" in speech) and (not "Pronoun" in speech) and (not "number" in speech):
        word_list.append(line.split()[0])

word_line = ",".join(word_list)

# ------------------------------------------------------------------------
from wordcloud import WordCloud
import matplotlib.pyplot as plt
#Fonts must be installed locally in Colab in advance
f_path = "BIZ-UDGothicB.ttc"
stop_words = ["https","imgur","net","jpg","com","so"]

wordcloud = WordCloud(
    font_path=f_path,
    width=1024, height=640,   # default width=400, height=200
    background_color="white",   # default=”black”
    stopwords=set(stop_words),
    max_words=350,   # default=200
    max_font_size=200,   #default=4
    min_font_size=5,   #default=4
    collocations = False   #default = True
    ).generate(word_line)
plt.figure(figsize=(18,15))
plt.imshow(wordcloud)
plt.axis("off") #Hide memory
plt.show()