{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:57:58.061370Z", "start_time": "2018-04-20T15:57:57.061631Z" }, "collapsed": true }, "outputs": [], "source": [ "import os\n", "import pandas as pd\n", "import gspread\n", "from oauth2client.service_account import ServiceAccountCredentials" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:57:58.201912Z", "start_time": "2018-04-20T15:57:58.191494Z" }, "collapsed": true }, "outputs": [], "source": [ "scope = ['https://spreadsheets.google.com/feeds']\n", "secrets = \"/Users/jeriwieringa/Dissertation/dev/code/secrets/dissertation-881847769b13.json\"\n", "credentials = ServiceAccountCredentials.from_json_keyfile_name(secrets, scope)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:57:59.096139Z", "start_time": "2018-04-20T15:57:58.930781Z" }, "collapsed": true }, "outputs": [], "source": [ "gc = gspread.authorize(credentials)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:58:02.540232Z", "start_time": "2018-04-20T15:58:00.243833Z" }, "collapsed": true }, "outputs": [], "source": [ "dts = gc.open('Topic Labels').sheet1\n", "frame = pd.DataFrame(dts.get_all_records())" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:58:02.574778Z", "start_time": "2018-04-20T15:58:02.542456Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
browser_topic_idmallet_topic_idtopic_categorytopic_labeltopic_prevalence_overalltopic_words
010missionsMissions (Canvassing Work)0.02981book canvasser order canvassing brother week s...
121nutritionNutrition (Recipes)0.00927cup water bread egg add cream salt flour milk ...
232scan_errorsScan Errors (OCR Errors)0.03660ing tion ment sign ness com ter tions great en...
343social_commentarySocial Commentary (Suffering and Death from Vi...0.03036death man blood men like life hand victim poor...
454spiritual_growthSpiritual Growth (Christian Life - Perseverance)0.02033shall lord unto faith thing hope trial christ ...
565sermonsSermons (Religious Practice)0.10352thing know think way man want let doe right go...
676bibleBible (Stories from Daniel and Babylonian Capt...0.01276king daniel babylon jerusalem lord kingdom neb...
787scan_errorsScan Errors (OCR Errors 2)0.04291ing lie review end herald ill tie ile tho good...
898spiritual_growthSpiritual Growth (Christian Behavior)0.01902prayer lord heart spirit blessing let church m...
9109reports_on_the_causeReports on the Cause (\"Progress of the Cause\")0.01000meeting church brother lord sabbath truth held...
\n", "
" ], "text/plain": [ " browser_topic_id mallet_topic_id topic_category \\\n", "0 1 0 missions \n", "1 2 1 nutrition \n", "2 3 2 scan_errors \n", "3 4 3 social_commentary \n", "4 5 4 spiritual_growth \n", "5 6 5 sermons \n", "6 7 6 bible \n", "7 8 7 scan_errors \n", "8 9 8 spiritual_growth \n", "9 10 9 reports_on_the_cause \n", "\n", " topic_label \\\n", "0 Missions (Canvassing Work) \n", "1 Nutrition (Recipes) \n", "2 Scan Errors (OCR Errors) \n", "3 Social Commentary (Suffering and Death from Vi... \n", "4 Spiritual Growth (Christian Life - Perseverance) \n", "5 Sermons (Religious Practice) \n", "6 Bible (Stories from Daniel and Babylonian Capt... \n", "7 Scan Errors (OCR Errors 2) \n", "8 Spiritual Growth (Christian Behavior) \n", "9 Reports on the Cause (\"Progress of the Cause\") \n", "\n", " topic_prevalence_overall topic_words \n", "0 0.02981 book canvasser order canvassing brother week s... \n", "1 0.00927 cup water bread egg add cream salt flour milk ... \n", "2 0.03660 ing tion ment sign ness com ter tions great en... \n", "3 0.03036 death man blood men like life hand victim poor... \n", "4 0.02033 shall lord unto faith thing hope trial christ ... \n", "5 0.10352 thing know think way man want let doe right go... \n", "6 0.01276 king daniel babylon jerusalem lord kingdom neb... \n", "7 0.04291 ing lie review end herald ill tie ile tho good... \n", "8 0.01902 prayer lord heart spirit blessing let church m... \n", "9 0.01000 meeting church brother lord sabbath truth held... " ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "frame[:10]" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:58:04.292602Z", "start_time": "2018-04-20T15:58:04.288585Z" }, "collapsed": true }, "outputs": [], "source": [ "topic2id = pd.Series(frame.topic_label.values,index=frame.browser_topic_id)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:58:09.130154Z", "start_time": "2018-04-20T15:58:09.124764Z" }, "collapsed": true }, "outputs": [], "source": [ "dict_view = {}\n", "for index, value in topic2id.iteritems():\n", " dict_view[\"{}\".format(index)] = \"{}\".format(value)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:58:14.125035Z", "start_time": "2018-04-20T15:58:14.119114Z" }, "collapsed": true }, "outputs": [], "source": [ "import json" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2018-04-20T15:58:14.711169Z", "start_time": "2018-04-20T15:58:14.704241Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"1\": \"Missions (Canvassing Work)\",\n", " \"10\": \"Reports on the Cause (\\\"Progress of the Cause\\\")\",\n", " \"100\": \"Theology (Christ and salvation)\",\n", " \"101\": \"Signs of the Times (Astronomy)\",\n", " \"102\": \"Conference Proceedings (Discussions)\",\n", " \"103\": \"Piety (Testimonies and Letters)\",\n", " \"104\": \"Theology (Sabbath Keeping)\",\n", " \"105\": \"Education (Educational Theory)\",\n", " \"106\": \"Meeting Reports (SDA Institutions)\",\n", " \"107\": \"Advertisements (Printing)\",\n", " \"108\": \"Health (Sanitariums - West Coast)\",\n", " \"109\": \"Advertisements (Subscriptions)\",\n", " \"11\": \"Theology (The Soul)\",\n", " \"110\": \"Social Commentary (Labor Organization and Disputes)\",\n", " \"111\": \"Missions (Prison Ministries)\",\n", " \"112\": \"Stories (Moralistic Stories)\",\n", " \"113\": \"Piety (Ocean as metaphor for life) \",\n", " \"114\": \"Education (SDA Colleges)\",\n", " \"115\": \"Theology (Sin)\",\n", " \"116\": \"Stories (Health-related)\",\n", " \"117\": \"Reports on the Cause (Field Reports)\",\n", " \"118\": \"Eschatology (Parable of Bridegroom and Tarrying Time)\",\n", " \"119\": \"Missions (News and Announcements)\",\n", " \"12\": \"Stories (Children's)\",\n", " \"120\": \"Apologetics (Geology and Evolution)\",\n", " \"121\": \"Advertisements (Pacific Press Books)\",\n", " \"122\": \"General Interest (Population and Statistics)\",\n", " \"123\": \"Nutrition (Grains)\",\n", " \"124\": \"Organization (Statistical Summaries)\",\n", " \"125\": \"Organization (Place names)\",\n", " \"126\": \"Reports on the Cause (History of SDA Work)\",\n", " \"127\": \"Bible (New Testament Lessons)\",\n", " \"128\": \"Missions\",\n", " \"129\": \"Conference Proceedings (Resolutions and Summaries)\",\n", " \"13\": \"Spiritual Growth (Christian Virtue - Development)\",\n", " \"130\": \"Apologetics (Other Religions)\",\n", " \"131\": \"Prophecy (Figures of Revelation)\",\n", " \"132\": \"Theology (Sabbath Keeping)\",\n", " \"133\": \"Organization (Funds and Figures - Pacific Union)\",\n", " \"134\": \"Advertisements (SDA Publications - early)\",\n", " \"135\": \"History (Sunday Observance)\",\n", " \"136\": \"Education (Teaching Plans)\",\n", " \"137\": \"Health (Rest)\",\n", " \"138\": \"Church & State (Religious Amendments and National Reform Association)\",\n", " \"139\": \"Advertisements\",\n", " \"14\": \"Sermons (Salvation Narrative)\",\n", " \"140\": \"Education (Sabbath Lessons)\",\n", " \"141\": \"Theology (the Fall)\",\n", " \"142\": \"Signs of the Times (Crime and Immorality)\",\n", " \"143\": \"Missions (South America)\",\n", " \"144\": \"Health (Ventilation)\",\n", " \"145\": \"Organization (Church Finances)\",\n", " \"146\": \"Eschatology (Sanctuary Doctrine)\",\n", " \"147\": \"Organization (Funds and Figures - Lake Union)\",\n", " \"148\": \"Nature (Animals)\",\n", " \"149\": \"Reports on the Cause (Building Projects)\",\n", " \"15\": \"Reports on the Cause (News and Field Reports)\",\n", " \"150\": \"Social Commentary (Commerce and Money)\",\n", " \"151\": \"Obituaries (Death Notices)\",\n", " \"152\": \"Obituaries (Death Notices)\",\n", " \"153\": \"Theology (Prayer)\",\n", " \"154\": \"Church & State (Arguments against Sabbath Laws)\",\n", " \"155\": \"Advertisements (Sanitariums)\",\n", " \"156\": \"Theology (the Sabbath Question)\",\n", " \"157\": \"Advertisements (Books and Literature)\",\n", " \"158\": \"Advertisements (Premium Bibles)\",\n", " \"159\": \"Signs of the Times (\\\"Catholic Threat\\\")\",\n", " \"16\": \"Politics (Religious Freedom)\",\n", " \"160\": \"Theology (Baptism and Sacraments)\",\n", " \"161\": \"Health (Habits)\",\n", " \"162\": \"Theology (Seventh-day Sabbath)\",\n", " \"163\": \"Organization (Lists of Donors and People)\",\n", " \"164\": \"Signs of the Times (War)\",\n", " \"165\": \"Stories (Moralistic Story - Family)\",\n", " \"166\": \"Stories (Theological)\",\n", " \"167\": \"Church & State (Sabbath Laws)\",\n", " \"168\": \"Theology (Ten Commandments)\",\n", " \"169\": \"Social Commentary (Temperance)\",\n", " \"17\": \"Piety (Poetry)\",\n", " \"170\": \"Spiritual Growth (Overcoming Temptation and Sin)\",\n", " \"171\": \"Eschatology (Events of the Last Days)\",\n", " \"172\": \"Spiritual Growth (Calls for Charity)\",\n", " \"173\": \"History (History in the Light of Prophecy)\",\n", " \"174\": \"Organization (Notices of Meetings and Appointments)\",\n", " \"175\": \"Nutrition (Foods and Recipes)\",\n", " \"176\": \"Organization (Columbia Union - Ohio - Activity Reports)\",\n", " \"177\": \"Missions (Africa)\",\n", " \"178\": \"Religious Commentary (Preaching and Ministry)\",\n", " \"179\": \"Bible (Quotations)\",\n", " \"18\": \"General Interest (News and Notes)\",\n", " \"180\": \"Signs of the Times (Global Unrest)\",\n", " \"181\": \"Social Commentary (Testimonies and Moral Commentary - largely EGW)\",\n", " \"182\": \"General Interest (Natural Disasters)\",\n", " \"183\": \"Eschatology (Description of Second Coming)\",\n", " \"184\": \"Health (Alcohol and Disease)\",\n", " \"185\": \"Health (Dangers of Alcohol)\",\n", " \"186\": \"Nature (Flora and Fauna)\",\n", " \"187\": \"Advertisements (Life Boat)\",\n", " \"188\": \"Health (Regular Medicine)\",\n", " \"189\": \"Bible (New Testament - Apostles)\",\n", " \"19\": \"Education (Medical Training Programs)\",\n", " \"190\": \"Theology (Righteousness by Faith)\",\n", " \"191\": \"Bible (Old Testament - Prophets)\",\n", " \"192\": \"Bible (Ten Commandments - Quoted)\",\n", " \"193\": \"Transportation (Trains)\",\n", " \"194\": \"Correspondence (Letters from Children)\",\n", " \"195\": \"Nutrition (Food - Meat)\",\n", " \"196\": \"Piety (Religious Comfort)\",\n", " \"197\": \"Theology (Holy Spirit)\",\n", " \"198\": \"Spiritual Growth (Christian Virtue)\",\n", " \"199\": \"Missions (Missions Work)\",\n", " \"2\": \"Nutrition (Recipes)\",\n", " \"20\": \"Advertisements (Transportation)\",\n", " \"200\": \"Eschatology (Second Coming)\",\n", " \"201\": \"Community News (Indiana)\",\n", " \"202\": \"Stories (Religious Stories)\",\n", " \"203\": \"Education (Farming and Agriculture)\",\n", " \"204\": \"Advertisements (Notices and Appointments)\",\n", " \"205\": \"Health (Eyes)\",\n", " \"206\": \"Missions (Sabbath School and Home Department)\",\n", " \"207\": \"Bible (Parables - Sower)\",\n", " \"208\": \"Advertisements (Bible Readers)\",\n", " \"209\": \"Education (Crafting Instructions)\",\n", " \"21\": \"Organization (Church Organization)\",\n", " \"210\": \"Reports on the Cause (\\\"Progress of the Cause\\\" - Reports on Local Reception)\",\n", " \"211\": \"Church & State (Religion Legislation and the Courts)\",\n", " \"212\": \"History (Reformation)\",\n", " \"213\": \"Organization (Report of Offerings)\",\n", " \"214\": \"Apologetics (Rebuttals and Responses)\",\n", " \"215\": \"Religious Commentary (Death)\",\n", " \"216\": \"Prophecy (Interpreting Dates and Time)\",\n", " \"217\": \"Health (Digestive System)\",\n", " \"218\": \"Missions (India)\",\n", " \"219\": \"Organization (Financial Report)\",\n", " \"22\": \"Politics (Principles of the Republic and their Violation)\",\n", " \"220\": \"Advertisements (Periodicals)\",\n", " \"221\": \"Missions (Home Missions)\",\n", " \"222\": \"Church & State (Religion in Public Education)\",\n", " \"223\": \"Bible (Quotations)\",\n", " \"224\": \"Community News (Lake Union & Tithe Reports)\",\n", " \"225\": \"Stories (Missionary stories)\",\n", " \"226\": \"Education\",\n", " \"227\": \"Community News (Southern Union)\",\n", " \"228\": \"Missions (Foreign Missions)\",\n", " \"229\": \"Meeting Reports (Tract and Missionary Societies)\",\n", " \"23\": \"Theology (Law and the New Testament)\",\n", " \"230\": \"Spiritual Growth (Christian Life)\",\n", " \"231\": \"Conference Proceedings (Resolutions Adopted)\",\n", " \"232\": \"Spiritual Growth (Christian Virtue)\",\n", " \"233\": \"Community News (Lake Union)\",\n", " \"234\": \"Reports on the Cause (\\\"Progress of the Cause\\\" - Report of Labor)\",\n", " \"235\": \"Theology (Christology)\",\n", " \"236\": \"Stories (Animals)\",\n", " \"237\": \"Periodicals (Publication Information - Michigan Titles)\",\n", " \"238\": \"Spiritual Growth (Christian Virtue)\",\n", " \"239\": \"Organization (Constitutions, Bylaws, and Articles of Incorporation)\",\n", " \"24\": \"Theology (Atonement)\",\n", " \"240\": \"Education (Lesson Plans for Sabbath Study)\",\n", " \"241\": \"Bible (Old Testament)\",\n", " \"242\": \"Missions (Of Missionary Ships)\",\n", " \"243\": \"Signs of the Times (Enforcement of Sunday Laws)\",\n", " \"244\": \"Spiritual Growth (Christian Education)\",\n", " \"245\": \"Periodicals (Washington, D.C. Titles)\",\n", " \"246\": \"Nutrition (Food)\",\n", " \"247\": \"Health (Dress Reform)\",\n", " \"248\": \"Organization (Divisions)\",\n", " \"249\": \"Missions (Missions Department)\",\n", " \"25\": \"Theology (Tithing)\",\n", " \"250\": \"Health (Exercise)\",\n", " \"26\": \"Organization (Notices of Meetings)\",\n", " \"27\": \"Religious Commentary (Spiritualism)\",\n", " \"28\": \"Signs of the Times (\\\"Eastern Question\\\")\",\n", " \"29\": \"Meeting Reports (Accounts of Camp Meetings and Conferences)\",\n", " \"3\": \"Scan Errors (OCR Errors)\",\n", " \"30\": \"Advertisements (Publications) \",\n", " \"31\": \"Politics (War and the State)\",\n", " \"32\": \"Health (Disease)\",\n", " \"33\": \"Missions (Canvassing Workers)\",\n", " \"34\": \"Theology (Great Controversy)\",\n", " \"35\": \"Advertisements (SDA Publications - mid)\",\n", " \"36\": \"Bible (Translations)\",\n", " \"37\": \"Theology (Sin)\",\n", " \"38\": \"Religious Commentary (Descriptions of Heaven)\",\n", " \"39\": \"Theology (God and Man)\",\n", " \"4\": \"Social Commentary (Suffering and Death from Vices)\",\n", " \"40\": \"Signs of the Times (Social Unrest)\",\n", " \"41\": \"Theology (Bible Interpretation)\",\n", " \"42\": \"Nutrition (Vegetables and Produce)\",\n", " \"43\": \"Church & State (Opposition to Religious Legislation)\",\n", " \"44\": \"Periodicals (Agencies for Magazine Ordering)\",\n", " \"45\": \"Missions (Chicago Mission)\",\n", " \"46\": \"Prophecy (Figures of Daniel) \",\n", " \"47\": \"Transportation (Train Travel)\",\n", " \"48\": \"Health (Infant Care)\",\n", " \"49\": \"Correspondence\",\n", " \"5\": \"Spiritual Growth (Christian Life - Perseverance)\",\n", " \"50\": \"Periodicals (Book Sales and Publishing Business)\",\n", " \"51\": \"Spiritual Growth (Calls to Christian Virtue - EGW)\",\n", " \"52\": \"Eschatology (Spreading the Third Angel's Message)\",\n", " \"53\": \"Bible (Parables - Wealth)\",\n", " \"54\": \"Bible (Parables - Talents)\",\n", " \"55\": \"Meeting Reports (Lake Union Conferences)\",\n", " \"56\": \"Community News (Columbia Union)\",\n", " \"57\": \"History (Historical Evidence for Reliability of Bible)\",\n", " \"58\": \"History (Religious Liberty in Anglo Political History)\",\n", " \"59\": \"Organization (The Chicago Mission)\",\n", " \"6\": \"Sermons (Religious Practice)\",\n", " \"60\": \"Correspondence (Letters from Readers)\",\n", " \"61\": \"Reports on the Cause (Account of Labors)\",\n", " \"62\": \"Meeting Reports (Descriptions of Denominational Events)\",\n", " \"63\": \"Nature (Landscapes)\",\n", " \"64\": \"Missions (Asia)\",\n", " \"65\": \"Health (Cleaning - teeth and household)\",\n", " \"66\": \"Education (Reading Course)\",\n", " \"67\": \"Nutrition (Diet)\",\n", " \"68\": \"Missions (Calls to Home Missions - EGW)\",\n", " \"69\": \"Advertisements (Health)\",\n", " \"7\": \"Bible (Stories from Daniel and Babylonian Captivity)\",\n", " \"70\": \"Organization (Funds - Harvest Ingathering)\",\n", " \"71\": \"Missions (Tract Distribution)\",\n", " \"72\": \"Meeting Reports (Southern Union Conferences)\",\n", " \"73\": \"Theology (the Dead)\",\n", " \"74\": \"Apologetics (Debates)\",\n", " \"75\": \"Missions (Foreign Missions)\",\n", " \"76\": \"General Interest (Military Culture)\",\n", " \"77\": \"General Interest (Architecture)\",\n", " \"78\": \"Bible (Old Testament Lessons)\",\n", " \"79\": \"Advertisements (Goods and Services)\",\n", " \"8\": \"Scan Errors (OCR Errors 2)\",\n", " \"80\": \"Stories (Character Development - Boys)\",\n", " \"81\": \"Bible (Biblical Exegesis)\",\n", " \"82\": \"Stories (Moralistic Stories - girls)\",\n", " \"83\": \"Organization (Travel Arrangements and Organization)\",\n", " \"84\": \"Social Commentary (Marriage and Divorce)\",\n", " \"85\": \"Spiritual Growth (Virtuousness and Industry)\",\n", " \"86\": \"Social Commentary (Amusement and Sport)\",\n", " \"87\": \"Signs of the Times (Growing Iniquity as in Days of Noah)\",\n", " \"88\": \"Advertisements (Subscription)\",\n", " \"89\": \"Bible (New Testament)\",\n", " \"9\": \"Spiritual Growth (Christian Behavior)\",\n", " \"90\": \"Social Commentary (Immigration and Non-English Language Concerns)\",\n", " \"91\": \"Missions (Relief for Indigent Children)\",\n", " \"92\": \"Advertisements (Music and Instruments)\",\n", " \"93\": \"Theology (Fate of the Wicked at Second Coming)\",\n", " \"94\": \"Health (Tuberculosis)\",\n", " \"95\": \"Spiritual Growth (Manners and Cheerfulness)\",\n", " \"96\": \"Health (Hydrotherapy)\",\n", " \"97\": \"General Interest (News and Catastrophic Events)\",\n", " \"98\": \"Health (Q & A)\",\n", " \"99\": \"Health (Coffee Tea Tobacco)\"\n", "}\n" ] } ], "source": [ "print(json.dumps(dict_view, indent=2, sort_keys=True))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python [conda env:dissertation2]", "language": "python", "name": "conda-env-dissertation2-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 2 }