In [ ]:
%conda install openjdk -y
%pip install pyspark==3.3.0
from IPython.core.display import HTML
HTML("<script>Jupyter.notebook.kernel.restart()</script>")

Collecting package metadata (current_repodata.json): done
Solving environment: done


==> WARNING: A newer version of conda exists. <==
  current version: 23.3.1
  latest version: 23.11.0

Please update conda by running

    $ conda update -n base -c defaults conda

Or to minimize the number of packages updated during conda update use

     conda install conda=23.11.0



## Package Plan ##

  environment location: /opt/conda

  added / updated specs:
    - openjdk


The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    ca-certificates-2023.08.22 |       h06a4308_0         123 KB
    certifi-2023.11.17         |  py310h06a4308_0         158 KB
    openjdk-11.0.13            |       h87a67e3_0       341.0 MB
    ------------------------------------------------------------
                                           Total:       341.3 MB

The following NEW packages will be INSTALLED:

  openjdk            pkgs/main/linux-64::openjdk-11.0.13-h87a67e3_0 

The following packages will be UPDATED:

  ca-certificates    conda-forge::ca-certificates-2023.7.2~ --> pkgs/main::ca-certificates-2023.08.22-h06a4308_0 
  certifi            conda-forge/noarch::certifi-2023.7.22~ --> pkgs/main/linux-64::certifi-2023.11.17-py310h06a4308_0 



Downloading and Extracting Packages
certifi-2023.11.17   | 158 KB    |                                       |   0% 
ca-certificates-2023 | 123 KB    |                                       |   0% 

openjdk-11.0.13      | 341.0 MB  |                                       |   0% 

openjdk-11.0.13      | 341.0 MB  | 1                                     |   0% 
ca-certificates-2023 | 123 KB    | ##################################### | 100% 
certifi-2023.11.17   | 158 KB    | ##################################### | 100% 

openjdk-11.0.13      | 341.0 MB  | 6                                     |   2% 

openjdk-11.0.13      | 341.0 MB  | ##1                                   |   6% 

openjdk-11.0.13      | 341.0 MB  | ###7                                  |  10% 

openjdk-11.0.13      | 341.0 MB  | ####9                                 |  13% 

openjdk-11.0.13      | 341.0 MB  | ######6                               |  18% 

openjdk-11.0.13      | 341.0 MB  | ########3                             |  23% 

openjdk-11.0.13      | 341.0 MB  | ##########1                           |  28% 

openjdk-11.0.13      | 341.0 MB  | ###########8                          |  32% 

openjdk-11.0.13      | 341.0 MB  | #############5                        |  37% 

openjdk-11.0.13      | 341.0 MB  | ###############3                      |  41% 

openjdk-11.0.13      | 341.0 MB  | #################                     |  46% 

openjdk-11.0.13      | 341.0 MB  | ##################6                   |  50% 

openjdk-11.0.13      | 341.0 MB  | ####################1                 |  54% 

openjdk-11.0.13      | 341.0 MB  | #####################6                |  58% 

openjdk-11.0.13      | 341.0 MB  | #######################               |  62% 

openjdk-11.0.13      | 341.0 MB  | ########################5             |  66% 

openjdk-11.0.13      | 341.0 MB  | ##########################            |  71% 

openjdk-11.0.13      | 341.0 MB  | ###########################8          |  75% 

openjdk-11.0.13      | 341.0 MB  | #############################5        |  80% 

openjdk-11.0.13      | 341.0 MB  | ###############################2      |  84% 

openjdk-11.0.13      | 341.0 MB  | ################################9     |  89% 

openjdk-11.0.13      | 341.0 MB  | ##################################7   |  94% 

openjdk-11.0.13      | 341.0 MB  | ####################################3 |  98% 

                                                                                
                                                                                

                                                                                
Preparing transaction: done
Verifying transaction: done
Executing transaction: done

Note: you may need to restart the kernel to use updated packages.
Collecting pyspark==3.3.0
  Using cached pyspark-3.3.0-py2.py3-none-any.whl
Collecting py4j==0.10.9.5 (from pyspark==3.3.0)
  Using cached py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)
Installing collected packages: py4j, pyspark
Successfully installed py4j-0.10.9.5 pyspark-3.3.0
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv

[notice] A new release of pip is available: 23.2.1 -> 23.3.1
[notice] To update, run: pip install --upgrade pip
Note: you may need to restart the kernel to use updated packages.
Out[ ]:
In [ ]:
import time
import sagemaker
import pandas as pd
from pyspark.sql import SparkSession
from pyspark.sql import functions as F
from pyspark.sql.functions import col, udf, explode
import matplotlib.pyplot as plt
from pyspark.ml.feature import Tokenizer, StopWordsRemover
from pyspark.sql.types import ArrayType, StringType
from nltk.stem import SnowballStemmer
from nltk.stem import WordNetLemmatizer
import nltk
import seaborn as sns
import matplotlib.pyplot as plt

sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml
sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml
In [ ]:
spark = (
    SparkSession.builder.appName("PySparkApp")
    .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:3.2.2")
    .config(
        "fs.s3a.aws.credentials.provider",
        "com.amazonaws.auth.ContainerCredentialsProvider",
    )
    .getOrCreate()
)

print(spark.version)

Warning: Ignoring non-Spark config property: fs.s3a.aws.credentials.provider

:: loading settings :: url = jar:file:/opt/conda/lib/python3.10/site-packages/pyspark/jars/ivy-2.5.0.jar!/org/apache/ivy/core/settings/ivysettings.xml

Ivy Default Cache set to: /root/.ivy2/cache
The jars for the packages stored in: /root/.ivy2/jars
org.apache.hadoop#hadoop-aws added as a dependency
:: resolving dependencies :: org.apache.spark#spark-submit-parent-10e7ae26-c26a-4a49-b1eb-94bccc5827cc;1.0
	confs: [default]
	found org.apache.hadoop#hadoop-aws;3.2.2 in central
	found com.amazonaws#aws-java-sdk-bundle;1.11.563 in central
:: resolution report :: resolve 344ms :: artifacts dl 21ms
	:: modules in use:
	com.amazonaws#aws-java-sdk-bundle;1.11.563 from central in [default]
	org.apache.hadoop#hadoop-aws;3.2.2 from central in [default]
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   2   |   0   |   0   |   0   ||   2   |   0   |
	---------------------------------------------------------------------
:: retrieving :: org.apache.spark#spark-submit-parent-10e7ae26-c26a-4a49-b1eb-94bccc5827cc
	confs: [default]
	0 artifacts copied, 2 already retrieved (0kB/16ms)

23/12/08 21:54:54 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).

3.3.0
In [ ]:
submissions = spark.read.parquet("s3a://dsan-6000-group-35/submissions.parquet", header=True)
comments = spark.read.parquet("s3a://dsan-6000-group-35/comments.parquet", header=True)

23/12/08 21:54:59 WARN MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-s3a-file-system.properties,hadoop-metrics2.properties
In [ ]:
submissions.show()

+----+-----+---------+------+--------------------+-------------------+--------------------+------------+--------------+-----+-------+--------+
|year|month|subreddit|    id|              author|        created_utc|                text|num_comments|num_crossposts|score|is_self|stickied|
+----+-----+---------+------+--------------------+-------------------+--------------------+------------+--------------+-----+-------+--------+
|2021|    6|    Music|o2omjb|     the_dionysian_1|2021-06-18 13:03:18|Bo Burnham - Welc...|        1656|             2|20968|  false|   false|
|2021|    6|    Music|o2onm6|       Seismic_Noise|2021-06-18 13:04:51|LIQUERUS Resident...|           0|             0|    2|  false|   false|
|2021|    6|    Music|o2oop3|SteveNewmanGuitarist|2021-06-18 13:06:31|Steve Newman - Rd...|           0|             0|    1|  false|   false|
|2021|    6|    Music|o2op0p|         Zoinksbeats|2021-06-18 13:07:01|(FREE) Playboi Ca...|           1|             0|    1|  false|   false|
|2021|    6|    Music|o2oqzq|          salem-1200|2021-06-18 13:10:02|Attention Required! |           1|             0|    1|  false|   false|
|2021|    6|    Music|o2org3|  BiscuitsAndTeaCups|2021-06-18 13:10:43|Biscuit Sundown -...|           0|             0|    1|  false|   false|
|2021|    6|    Music|o2otxk|          Funkedalic|2021-06-18 13:14:32|XTC - That's Real...|           0|             0|    2|  false|   false|
|2021|    6|    Music|o2ou3y|          Taffia1977|2021-06-18 13:14:46|Armand the Chemis...|           1|             0|    2|  false|   false|
|2021|    6|    Music|o2ouv2|      TorontoRichard|2021-06-18 13:15:53|Skip Stompin’ Ric...|           0|             0|    0|  false|   false|
|2021|    6|    Music|o2oww6|    ShiftAxisRecords|2021-06-18 13:18:51|[Powerful Vocal H...|           1|             0|    1|  false|   false|
|2021|    6|    Music|o2oxg3|            zapomaze|2021-06-18 13:19:39|Zapo Maze - All T...|           2|             0|    0|  false|   false|
|2021|    6|    Music|o2oya4|    ShiftAxisRecords|2021-06-18 13:20:50|Andrew Ford – Cra...|           1|             0|    1|  false|   false|
|2021|    6|    Music|o2ozau|       sherlockinggg|2021-06-18 13:22:15|Big Four Whicheve...|           7|             0|    1|   true|   false|
|2021|    6|    Music|o2ozku|           McDoofyto|2021-06-18 13:22:37|The Fourth King -...|           1|             0|    1|  false|   false|
|2021|    6|    Music|o2p4m3|         JosiahSoren|2021-06-18 13:29:43|22 Lessons I lear...|           6|             0|   28|   true|   false|
|2021|    6|    Music|o2p7s4|           lucalampe|2021-06-18 13:34:07|A synthwave and 8...|           1|             0|    1|  false|   false|
|2021|    6|    Music|o2p8tu|        thiccnations|2021-06-18 13:35:35|        Piano music |           1|             0|    1|  false|   false|
|2021|    3|    Music|m2t49n|           Patinhooo|2021-03-11 15:43:32|Chill vibes music...|           0|             0|    1|   true|   false|
|2021|    3|    Music|m2t6x5| Scary_Teaching_4825|2021-03-11 15:46:45|yesyeskrishnacall...|           1|             0|    1|  false|   false|
|2021|    3|    Music|m2t8l5|           Patinhooo|2021-03-11 15:48:42|Chill music https...|           0|             0|    0|   true|   false|
+----+-----+---------+------+--------------------+-------------------+--------------------+------------+--------------+-----+-------+--------+
only showing top 20 rows
In [ ]:
ts_df = submissions.filter(col('text')
                           .rlike('(?i)' + '|'
                                  .join(['Taylor Swift', 'Taylor', 'Swift', 'Swizzle', 'Tay', 'Swiftie','Swifty','T-Swift','Miss Americana','Cruel Summer','All Too Well'])))
num_rows = ts_df.count()
num_cols = len(ts_df.columns)
print(f"The DataFrame has {num_rows} rows and {num_cols} columns.")

[Stage 2:============================================>              (3 + 1) / 4]

The DataFrame has 25421 rows and 12 columns.

Submission Author Engagement¶

In [ ]:
author_engagement = ts_df.groupBy('author') \
    .agg(
        F.count('id').alias('num_posts'),
        F.sum('num_comments').alias('total_comments'),
        F.sum('score').alias('total_score')
    ) \
    .orderBy(F.col('total_comments').desc(), F.col('total_score').desc())
author_engagement_pd = author_engagement.toPandas()
In [ ]:
author_engagement_summary = author_engagement.agg(
    F.sum('num_posts').alias('total_num_posts_submissions'),
    F.sum("total_comments").alias("total_comments_submissions")
)
author_engagement_summary.show()
author_engagement_summary.toPandas().to_csv('../../data/csv/eda/author_engagement_summary.csv',index=False)

                                                                                

+---------------------------+--------------------------+
|total_num_posts_submissions|total_comments_submissions|
+---------------------------+--------------------------+
|                      25421|                   1367202|
+---------------------------+--------------------------+
In [ ]:
top_authors = author_engagement_pd.sort_values(by="num_posts", ascending=False).head(15)
top_comments = author_engagement_pd.sort_values(by="total_comments", ascending=False).head(15)
top_scores = author_engagement_pd.sort_values(by="total_score", ascending=False).head(15)


fig, axes = plt.subplots(3, 1, figsize=(8, 18), sharex=False, constrained_layout=True)
fig.suptitle('Top 15 Fanbase Engagement Metrics in Submission', fontsize=16)
sns.barplot(ax=axes[0], x='author', y='num_posts', data=top_authors, palette="viridis")
axes[0].set_title('Number of Posts by Author')
axes[0].set_ylabel('Number of Posts')
axes[0].tick_params(axis='x', rotation=90)
    
sns.barplot(ax=axes[1], x='author', y='total_comments', data=top_comments, palette="magma")
axes[1].set_title('Total Comments by Author')
axes[1].set_ylabel('Total Comments')
axes[1].tick_params(axis='x', rotation=90)

sns.barplot(ax=axes[2], x='author', y='total_score', data=top_scores, palette="cubehelix")
axes[2].set_title('Total Score by Author')
axes[2].set_ylabel('Total Score')
axes[2].tick_params(axis='x', rotation=90)

for ax in axes:
    for spine in ax.spines.values():
        spine.set_visible(False)

plt.savefig('../../data/plots/eda/elliot_engagement_metrics_submission.png',dpi=1200)
plt.show()
In [ ]:
top_authors = author_engagement_pd.sort_values(by="num_posts", ascending=False).head(15)
top_comments = author_engagement_pd.sort_values(by="total_comments", ascending=False).head(15)
top_scores = author_engagement_pd.sort_values(by="total_score", ascending=False).head(15)
In [ ]:
top_authors.to_csv("top_authors.csv",index = False)
top_comments.to_csv("top_comments.csv",index = False)
top_scores.to_csv("top_scores.csv",index = False)

Submission Word Count¶

In [ ]:
nltk.download('wordnet')
nltk.download('omw-1.4')

stemmer = SnowballStemmer(language='english')
lemmatizer = WordNetLemmatizer()
tokenizer = Tokenizer(inputCol='text', outputCol='words')
remover = StopWordsRemover(inputCol='words', outputCol='filtered')

def stem_words(words):
    return [stemmer.stem(word) for word in words]
def lemmatize_words(words):
    return [lemmatizer.lemmatize(word) for word in words]
stemming_udf = udf(stem_words, ArrayType(StringType()))
lemmatization_udf = udf(lemmatize_words, ArrayType(StringType()))

ts_df_words = tokenizer.transform(ts_df)
ts_df_filtered = remover.transform(ts_df_words)
ts_df_stemmed = ts_df_filtered.withColumn("stemmed", stemming_udf("filtered"))
ts_df_lemmatized = ts_df_stemmed.withColumn("lemmatized", lemmatization_udf("stemmed"))

word_frequency = ts_df_lemmatized.withColumn('word', explode('lemmatized')) \
    .groupBy('word').count() \
    .orderBy('count', ascending=False)
word_frequency_pd = word_frequency.toPandas()

[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data] Downloading package omw-1.4 to /root/nltk_data...
In [ ]:
def is_english_word(word):
    return word.isalpha() and len(word) >= 4 

top_30_words = (
    word_frequency_pd[word_frequency_pd['word'].apply(is_english_word)]
    .nlargest(30, 'count')
    .reset_index(drop=True)
)
In [ ]:
top_30_words.to_csv("top_30_words_submissions.csv",index = False)
In [ ]:
ts_comments = comments.filter(col('body')
                           .rlike('(?i)' + '|'
                                  .join(['Taylor Swift', 'Taylor', 'Swift', 'Swizzle', 'Tay', 'Swiftie','Swifty','T-Swift','Miss Americana','Cruel Summer','All Too Well'])))

stemmer = SnowballStemmer(language='english')
lemmatizer = WordNetLemmatizer()
tokenizer = Tokenizer(inputCol='body', outputCol='words')
remover = StopWordsRemover(inputCol='words', outputCol='filtered')

def stem_words(words):
    return [stemmer.stem(word) for word in words]
def lemmatize_words(words):
    return [lemmatizer.lemmatize(word) for word in words]

stemming_udf = udf(stem_words, ArrayType(StringType()))
lemmatization_udf = udf(lemmatize_words, ArrayType(StringType()))

ts_comments = tokenizer.transform(ts_comments)
ts_comments = remover.transform(ts_comments)
ts_comments = ts_comments.withColumn("stemmed", stemming_udf("filtered"))
ts_comments = ts_comments.withColumn("lemmatized", lemmatization_udf("stemmed"))

comments_word_frequency = ts_comments.withColumn('word', explode('lemmatized')) \
    .groupBy('word').count() \
    .orderBy('count', ascending=False)

comments_word_frequency_pd = comments_word_frequency.toPandas()

top_30_words = (
    comments_word_frequency_pd[comments_word_frequency_pd['word'].apply(is_english_word)]
    .nlargest(30, 'count')
    .reset_index(drop=True)
)

top_30_words.to_csv("top_30_words_comments.csv",index = False)
In [ ]:
plt.figure(figsize=(8, 6),constrained_layout=True)
sns.barplot(x='word', y='count', data=top_30_words, palette="viridis")
plt.title('Top 30 Most Frequent Words Used in Submission')
plt.ylabel('Word Count')
plt.xticks(rotation=90)
for spine in plt.gca().spines.values():
    spine.set_visible(False)
plt.savefig('../../data/plots/eda/elliot_top30_words_submission.png',dpi=1200)
plt.show()
In [ ]:
comments.show()

+----+-----+-----------+-------+------------------+---------+----------+-------------------+--------------------+-----+------+----------------+
|year|month|  subreddit|     id|            author|  link_id| parent_id|        created_utc|                body|score|gilded|controversiality|
+----+-----+-----------+-------+------------------+---------+----------+-------------------+--------------------+-----+------+----------------+
|2022|   10|TaylorSwift|irf7cfk|    realscoutfinch|t3_xwui3g|t1_iracq55|2022-10-07 16:46:13|This this this! I...|    1|     0|               0|
|2022|   10|      Music|irf7cqw|        Alouitious|t3_xxz930|t1_irf6yfd|2022-10-07 16:46:17|4. Once you've bu...|    4|     0|               0|
|2022|   10|      Music|irf7dpm|     ToastedSimian|t3_xy1kr4|t1_ireuzta|2022-10-07 16:46:29|Sadly, I'm a work...|    1|     0|               0|
|2022|   10|      Music|irf7drt|        tommykiddo|t3_xxz930|t1_irf39w8|2022-10-07 16:46:30|Ultimate Guitar h...|    3|     0|               0|
|2022|   10|      Music|irf7ghz|          monkee67|t3_xy0jgc| t3_xy0jgc|2022-10-07 16:47:03|MEH  \n\nBTW this...|   -2|     0|               1|
|2022|   10|TaylorSwift|irf7gn3|       dietrichs90|t3_xt3tob| t3_xt3tob|2022-10-07 16:47:05|Taylors caption t...|   21|     0|               0|
|2022|   10|      Music|irf7jai|        dogsarefun|t3_xxrwic|t1_irdprwc|2022-10-07 16:47:38|Also, Aesop Rock ...|    0|     0|               0|
|2022|   10|TaylorSwift|irf7jgy|      evergreenkat|t3_xxy08f| t3_xxy08f|2022-10-07 16:47:40|The whole left co...|    2|     0|               0|
|2022|   10|TaylorSwift|irf7qr0|       sapphicsato|t3_xxr3aj| t3_xxr3aj|2022-10-07 16:49:08|IWAASPIWTWWGROMBF...|    1|     0|               0|
|2022|   10|      Music|irf7txv|           Kidspud|t3_xy0jgc|t1_irepwh7|2022-10-07 16:49:47|I agree 100%. I f...|   17|     0|               0|
|2022|   10|TaylorSwift|irf7uk8|      hannahberrie|t3_xtuyhw|t1_irc8u25|2022-10-07 16:49:54|               Whelp|    1|     0|               0|
|2022|   10|TaylorSwift|irf7vm7|     AutoModerator|t3_xy42ef| t3_xy42ef|2022-10-07 16:50:07|**All posts are a...|    1|     0|               0|
|2022|   10|      Music|irf813a|       okdude23232|t3_91z3j5|t1_ino5ijx|2022-10-07 16:51:15|people who call t...|    1|     0|               0|
|2022|   10|TaylorSwift|irf8464|            vlarek|t3_xt3tob|t1_irf7rn4|2022-10-07 16:51:53|Grey's aired last...|    7|     0|               0|
|2022|   10|TaylorSwift|irf86gd|       robynnc1290|t3_xt3tob|t1_irf7gn3|2022-10-07 16:52:22|I think there’s s...|    8|     0|               0|
|2022|   10|      Music|irf86z7|        -n0isyb0y-|t3_lzmzv9| t3_lzmzv9|2022-10-07 16:52:29|I am not sure but...|    1|     0|               0|
|2022|   10|      Music|irf8aqn|   drewisawesome14|t3_xx7syx|t1_irdn777|2022-10-07 16:53:16|That’s interestin...|    1|     0|               0|
|2022|   10|      Music|irf8bch|JoeCorsonStageDeli|t3_xy0jgc| t3_xy0jgc|2022-10-07 16:53:23|Used to really re...|    7|     0|               0|
|2022|   10|TaylorSwift|irf8hei|             lom41|t3_xxy08f|t1_irelrrs|2022-10-07 16:54:38|I love the anti-h...|    5|     0|               0|
|2022|   10|TaylorSwift|irf8ijo|      indievibes23|t3_xxt7wd| t3_xxt7wd|2022-10-07 16:54:52|I’M SO EXCITED! I...|    3|     0|               0|
+----+-----+-----------+-------+------------------+---------+----------+-------------------+--------------------+-----+------+----------------+
only showing top 20 rows
In [ ]:
tsc_df = comments.filter(col('body')
                           .rlike('(?i)' + '|'
                                  .join(['Taylor Swift', 'Taylor', 'Swift', 'Swizzle', 'Tay', 'Swiftie','Swifty','T-Swift','Miss Americana','Cruel Summer','All Too Well'])))
num_rows = tsc_df.count()
num_cols = len(tsc_df.columns)
print(f"The DataFrame has {num_rows} rows and {num_cols} columns.")

[Stage 32:====================================================>   (14 + 1) / 15]

The DataFrame has 357719 rows and 12 columns.

Comment Author Engagement¶

In [ ]:
author_engagement = tsc_df.groupBy('author') \
    .agg(
        F.count('id').alias('num_comments'),
        F.sum('score').alias('comment_score')
    ) \
    .orderBy(F.col('num_comments').desc(), F.col('comment_score').desc())
author_engagement_pd = author_engagement.toPandas()
author_engagement_pd = author_engagement_pd.iloc[1:]
In [ ]:
author_engagement_summary = author_engagement.agg(
    F.sum('num_comments').alias('total_comments_comments'),
    F.sum("comment_score").alias("total_comment_score_comments")
)
author_engagement_summary.show()
author_engagement_summary.toPandas().to_csv('../../data/csv/eda/author_engagement_comments_summary.csv',index=False)

                                                                                

+-----------------------+----------------------------+
|total_comments_comments|total_comment_score_comments|
+-----------------------+----------------------------+
|                 357719|                     5670273|
+-----------------------+----------------------------+
In [ ]:
top_authors = author_engagement_pd.sort_values(by="num_comments", ascending=False).head(15)
top_comment_score = author_engagement_pd.sort_values(by="comment_score", ascending=False).head(15)
In [ ]:
top_authors.to_csv("top_authors_comments.csv",index = False)
top_comment_score.to_csv("top_authors_scores.csv",index = False)
In [ ]:
fig, axes = plt.subplots(2, 1, figsize=(8, 12), sharex=False, constrained_layout=True)
fig.suptitle('Top 15 Fanbase Engagement Metrics in Comment', fontsize=16)
sns.barplot(ax=axes[0], x='author', y='num_comments', data=top_authors, palette="viridis")
axes[0].set_title('Number of Comments by Author')
axes[0].set_ylabel('Number of Comments')
axes[0].tick_params(axis='x', rotation=90)
    
sns.barplot(ax=axes[1], x='author', y='comment_score', data=top_comment_score, palette="magma")
axes[1].set_title('Top Upvoted Comment Scores by Author')
axes[1].set_ylabel('Comment Scores')
axes[1].tick_params(axis='x', rotation=90)

for ax in axes:
    for spine in ax.spines.values():
        spine.set_visible(False)
        
plt.savefig('../../data/plots/eda/elliot_engagement_metrics_comment.png',dpi=1200)
plt.show()

Comment Word Count¶

In [ ]:
nltk.download('wordnet')
nltk.download('omw-1.4')

stemmer = SnowballStemmer(language='english')
lemmatizer = WordNetLemmatizer()
tokenizer = Tokenizer(inputCol='body', outputCol='words')
remover = StopWordsRemover(inputCol='words', outputCol='filtered')

def stem_words(words):
    return [stemmer.stem(word) for word in words]
def lemmatize_words(words):
    return [lemmatizer.lemmatize(word) for word in words]
stemming_udf = udf(stem_words, ArrayType(StringType()))
lemmatization_udf = udf(lemmatize_words, ArrayType(StringType()))

ts_df_words = tokenizer.transform(tsc_df)
ts_df_filtered = remover.transform(ts_df_words)
ts_df_stemmed = ts_df_filtered.withColumn("stemmed", stemming_udf("filtered"))
ts_df_lemmatized = ts_df_stemmed.withColumn("lemmatized", lemmatization_udf("stemmed"))

word_frequency = ts_df_lemmatized.withColumn('word', explode('lemmatized')) \
    .groupBy('word').count() \
    .orderBy('count', ascending=False)
word_frequency_pd = word_frequency.toPandas()

[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!
[nltk_data] Downloading package omw-1.4 to /root/nltk_data...
[nltk_data]   Package omw-1.4 is already up-to-date!
In [ ]:
def is_english_word(word):
    return word.isalpha() and len(word) >= 4 

top_30_words = (
    word_frequency_pd[word_frequency_pd['word'].apply(is_english_word)]
    .nlargest(30, 'count')
    .reset_index(drop=True)
)
In [ ]:
plt.figure(figsize=(8, 6),constrained_layout=True)
sns.barplot(x='word', y='count', data=top_30_words, palette="viridis")
plt.title('Top 30 Most Frequent Words Used in Comment')
plt.ylabel('Word Count')
plt.xticks(rotation=90)
for spine in plt.gca().spines.values():
    spine.set_visible(False)
    
plt.savefig('../../data/plots/eda/elliot_top30_words_comment.png',dpi=1200)
plt.show()