472 lines
14 KiB
Plaintext
472 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6ad03636-9c4b-48f1-b85d-6ae8527d9984",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"###################################################\n",
|
|
"#___ ___ _ _ _ #\n",
|
|
"#| \\/ | | | | | | | #\n",
|
|
"#| . . | __ _ ___| |_ ___ __| | __ _| |_ __ _ #\n",
|
|
"#| |\\/| |/ _` / __| __/ _ \\ / _` |/ _` | __/ _` | #\n",
|
|
"#| | | | (_| \\__ \\ || (_) | (_| | (_| | || (_| | #\n",
|
|
"#\\_| |_/\\__,_|___/\\__\\___/ \\__,_|\\__,_|\\__\\__,_| #\n",
|
|
"################################################### \n",
|
|
"# This script loads data from the public and local tag timeline.\n",
|
|
"# Please fill in the correct path, where your data \n",
|
|
"# is stored.\n",
|
|
"#######################################################\n",
|
|
"# At the current stage, mastodata data loaders are meant\n",
|
|
"# to briefly explore collected data, filtered and then \n",
|
|
"# exported. You can filter by date and hashtag, for \n",
|
|
"# public and local timeline.\n",
|
|
"#######################################################\n",
|
|
"# There are three analytical sections in this notebook\n",
|
|
"# for each timeline.\n",
|
|
"# 1. Descriptive overview of the material\n",
|
|
"# 2. Co-Hashtags\n",
|
|
"# 3. Federation\n",
|
|
"#######################################################\n",
|
|
"\n",
|
|
"import os\n",
|
|
"import json\n",
|
|
"import pandas as pd\n",
|
|
"from glob import glob\n",
|
|
"import matplotlib.pyplot as plt"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e8bc49e9-341e-4507-9221-a1bf590921e5",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Data for 'timeline_public'"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "de8f017c-3d2c-4c97-a9bc-c586aebeb20c",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Loading data: Public data is loaded to variable 'df_public'.\n",
|
|
"# Given path must lead directly to json files from the collection and end with '/*'\n",
|
|
"# E.g. 'YOUR/PATH/timeline_public/*' \n",
|
|
"path_public = \n",
|
|
"\n",
|
|
"data_public = []\n",
|
|
"for fi in glob(path_public):\n",
|
|
" if os.path.isfile(fi) and fi.endswith('.json'):\n",
|
|
" with open(fi, 'r') as infile:\n",
|
|
" data_public += json.load(infile)\n",
|
|
" \n",
|
|
"df_public = pd.json_normalize(data_public)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "e64f6782-35a9-406d-9cca-40ac40f35127",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Stats"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2de303b6-25d7-4212-9580-54cbd9c43cfe",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Add average posts per MAU \n",
|
|
"print('Posts in Total:', len(df_public)) \n",
|
|
"print('Amount of individual users:',len(pd.unique(df_public['account.username'])))\n",
|
|
"print('Average posts per user:',len(df_public)/len(pd.unique(df_public['account.username'])))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9e8e561d-1783-4cd5-accd-1a8f3de6c973",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Show amount of toots sorted by (calendar) week\n",
|
|
"# Note: Week twenty three is not fully part of the data set. Therefore that week is incomplete. Keep that in mind during analysis.\n",
|
|
"df_public_dates = df_public[\"created_at\"].astype(\"datetime64\")\n",
|
|
"df_public_dates.groupby(df_public_dates.dt.isocalendar().week).count().plot(kind=\"bar\")\n",
|
|
"\n",
|
|
"## Keep in mind, week 23 was not fully archived. "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b54a52b0-da74-4119-a08b-a6b35bb1bb6d",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Postings distributed by weekdays \n",
|
|
"df_public_dates.groupby(df_public_dates.dt.isocalendar().day).count().plot(kind=\"bar\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3c9c0fe9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "1fd7e75d-83e3-4073-b39f-16b93b82a694",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Date specific filtering\n",
|
|
"############################\n",
|
|
"# Please insert the time frame by \n",
|
|
"# specifying a start date and an end date \n",
|
|
"\n",
|
|
"#start_date = \n",
|
|
"#end_date = \n",
|
|
"\n",
|
|
"\n",
|
|
"#mask = (df_public['created_at'] > start_date) & (df_public['created_at'] <= end_date)\n",
|
|
"\n",
|
|
"# Uncommment below to overwrite existing data frame\n",
|
|
"\n",
|
|
"#df_public = df_public.loc[mask]\n",
|
|
"\n",
|
|
"# Uncomment below to create subset called 'filtered_df'\n",
|
|
"#filtered_df = df_public.loc[mask]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "8497e393-ac7f-45e8-824e-3cbc874fb2e2",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Hashtag analysis"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d299dd8a-ca4c-4069-94d7-b4dfd5540897",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Um die Tags zu analysieren macht es Sinn eine Eigene Tabelle zu erstellen, die sämtliche Tags und Informationen zu diesen enthält. \n",
|
|
"# Hier ist ein Beispiel um eine solche Tabelle zu erstellen. Neben dem Tag wird die Toot ID, das Toot created_at sowie der account in jeder Zeile mitgeführt. \n",
|
|
"# Das kann man je nach analytischem Interesse erweitern. Für Datensparsamkeit beim speichern würde die Toot ID reichen. Den rest kann man mit der anderen Tabelle \n",
|
|
"# auch joinen mit pd.merge(df, tags, left_on='id', right_on='toot_id').\n",
|
|
"\n",
|
|
"tags_public = []\n",
|
|
"for row in df_public.iterrows():\n",
|
|
" for tag in row[1]['tags']:\n",
|
|
" tag['toot_id'] = row[1]['id']\n",
|
|
" tag['toot_created_at'] = row[1]['created_at']\n",
|
|
" tag['toot_account.id'] = row[1]['account.id'] \n",
|
|
" tags_public.append(tag)\n",
|
|
"\n",
|
|
"tags_public = pd.json_normalize(tags_public)\n",
|
|
"# For a first look\n",
|
|
"tags_public.head(5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7c9d406d-458e-4cc4-b6b7-85434e66dc98",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Hashtags sorted by frequency\n",
|
|
"print(tags_public.groupby('name').size().sort_values(ascending=False))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2345142d-b9a2-4011-822e-81dca9000034",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Show Top 10 Toots as bar plot\n",
|
|
"#find values with top 10 occurrences in 'my_column'\n",
|
|
"top_10 = (tags_public['name'].value_counts()).iloc[:10]\n",
|
|
"top_10.plot(kind='bar')\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "80d37aab",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Show top 25 Users by amount of posts\n",
|
|
"\n",
|
|
"top_25_users = (df_public['account.username'].value_counts()).iloc[:25]\n",
|
|
"top_25_users.plot(kind='bar')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "262aa947-9b5c-4550-8b36-b09ed81a2ae7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Extract toots containing a certain hashtag\n",
|
|
"# Insert hashtags tags to import in a seperate data set\n",
|
|
"# Hashtag name must be inserted within quotes!\n",
|
|
"\n",
|
|
"my_tags_public = []\n",
|
|
"\n",
|
|
"df_my_tags_public = tags_public[tags_public['name'].isin(my_tags_public)]\n",
|
|
"\n",
|
|
"df_my_tags_public.head() \n",
|
|
"\n",
|
|
"# uncomment the following command to create a csv-file for further examination\n",
|
|
"\n",
|
|
"#df_my_tags_public.to_csv('exports/my_tags_public.csv')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "adab07a1-1ba4-407a-8151-cc890fe6603a",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Data for 'timeline_local'"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3131ef1f-6f8d-4dcb-a801-553019a79e90",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Loading data: Public data is loaded to variable 'df_public'.\n",
|
|
"# ATTENTION: In this environment relative paths don't work with glob. \n",
|
|
"path_local = '/var/jupyter-data/jupyter-mastodata/data/mastodon.social/timeline_local/*'\n",
|
|
"\n",
|
|
"data_local = []\n",
|
|
"for fi in glob(path_local):\n",
|
|
" if os.path.isfile(fi) and fi.endswith('.json'):\n",
|
|
" with open(fi, 'r') as infile:\n",
|
|
" data_local += json.load(infile)\n",
|
|
" \n",
|
|
"df_local = pd.json_normalize(data_local)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "10a6cafb-861e-4c8b-bc1f-e0fd5f67e7ce",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Stats"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c5133131-1a74-4c61-8f3c-4044612a0394",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Add average posts per MAU\n",
|
|
"print('Posts in Total:', len(df_local)) \n",
|
|
"print('Amount of individual users:',len(pd.unique(df_local['account.username'])))\n",
|
|
"print('Average posts per user:',len(df_local)/len(pd.unique(df_local['account.username'])))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "c3c20029-0d0a-48e8-9632-5bab4b21d99b",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Change datetype of 'created_at'\n",
|
|
"df_local_dates = df_local[\"created_at\"].astype(\"datetime64\")\n",
|
|
"# Show amount of toots sorted by (calendar) week\n",
|
|
"df_local_dates.groupby(df_local_dates.dt.isocalendar().week).count().plot(kind=\"bar\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "dfbd773c-7cbc-4fe8-8343-934fe747faa2",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Postings distributed by weekdays\n",
|
|
"df_local_dates.groupby(df_local_dates.dt.isocalendar().day).count().plot(kind=\"bar\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5b99b490",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Show top 25 Users by amount of posts\n",
|
|
"\n",
|
|
"top_25_users = (df_local['account.username'].value_counts()).iloc[:25]\n",
|
|
"top_25_users.plot(kind='bar')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "f24fccb8-0cb6-4241-89a8-36cd714e235e",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Hashtag analysis"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3f0c3c70-aeb5-4359-9319-f3467a8b7162",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Um die Tags zu analysieren macht es Sinn eine Eigene Tabelle zu erstellen, die sämtliche Tags und Informationen zu diesen enthält. \n",
|
|
"# Hier ist ein Beispiel um eine solche Tabelle zu erstellen. Neben dem Tag wird die Toot ID, das Toot created_at sowie der account in jeder Zeile mitgeführt. \n",
|
|
"# Das kann man je nach analytischem Interesse erweitern. Für Datensparsamkeit beim speichern würde die Toot ID reichen. Den rest kann man mit der anderen Tabelle \n",
|
|
"# auch joinen mit pd.merge(df, tags, left_on='id', right_on='toot_id').\n",
|
|
"\n",
|
|
"tags_local = []\n",
|
|
"for row in df_local.iterrows():\n",
|
|
" for tag in row[1]['tags']:\n",
|
|
" tag['toot_id'] = row[1]['id']\n",
|
|
" tag['toot_created_at'] = row[1]['created_at']\n",
|
|
" tag['toot_account.id'] = row[1]['account.id'] \n",
|
|
" tags_local.append(tag)\n",
|
|
"\n",
|
|
"tags_local = pd.json_normalize(tags_local)\n",
|
|
"tags_local.head(5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "561054f2-d8db-47d9-aa15-c0ea5f06388d",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Hashtags sorted by frequency\n",
|
|
"tags_local.groupby('name').size().sort_values(ascending=False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "69bba3a4-03b0-4cd1-8172-3418ad06b446",
|
|
"metadata": {
|
|
"tags": []
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Show Top 10 Toots as bar plot\n",
|
|
"#find values with top 10 occurrences in 'my_column'\n",
|
|
"top_10_tags = (tags_local['name'].value_counts()).iloc[:10]\n",
|
|
"top_10_tags.plot(kind='bar')\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "90785572",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "54ed0e60-9349-4cf1-b987-04aa3aa07037",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Extract toots containing a certain hashtag\n",
|
|
"# Insert hashtags tags to import in a seperate data set\n",
|
|
"# Hashtag name must be inserted within quotes!\n",
|
|
"\n",
|
|
"my_tags_local = []\n",
|
|
"\n",
|
|
"df_my_tags_local = tags_local[tags_local['name'].isin(my_tags_local)]\n",
|
|
"\n",
|
|
"df_my_tags_local.head() \n",
|
|
"\n",
|
|
"# uncomment the following command to create a csv-file for further examination\n",
|
|
"\n",
|
|
"#df_my_tags_local.to_csv('exports/my_tags_local.csv')"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|