<meta charset="utf-8" />

<meta name="generator" content="pandoc" />

<meta http-equiv="X-UA-Compatible" content="IE=EDGE" />

<meta name="author" content="Andreas Niekler, Gregor Wiedemann" />

<meta name="date" content="2020-10-08" />

<title>Text mining in R for the social sciences and digital humanities</title>

<script src="site_libs/jquery-1.11.3/jquery.min.js"></script>

<meta name="viewport" content="width=device-width, initial-scale=1" />

<link href="site_libs/bootstrap-3.3.5/css/united.min.css" rel="stylesheet" />

<script src="site_libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>

<script src="site_libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>

<script src="site_libs/bootstrap-3.3.5/shim/respond.min.js"></script>

<script src="site_libs/jqueryui-1.11.4/jquery-ui.min.js"></script>

<link href="site_libs/tocify-1.9.1/jquery.tocify.css" rel="stylesheet" />

<script src="site_libs/tocify-1.9.1/jquery.tocify.js"></script>

<script src="site_libs/navigation-1.1/tabsets.js"></script>

<link href="site_libs/highlightjs-9.12.0/default.css" rel="stylesheet" />

<script src="site_libs/highlightjs-9.12.0/highlight.js"></script>

<link href="site_libs/ionicons-2.0.1/css/ionicons.min.css" rel="stylesheet" />

<style type="text/css">code{white-space: pre;}</style>

<style type="text/css">

pre:not([class]) {

background-color: white;

}

<script type="text/javascript">

if (window.hljs) {

hljs.configure({languages: []});

hljs.initHighlightingOnLoad();

if (document.readyState && document.readyState === "complete") {

window.setTimeout(function() { hljs.initHighlighting(); }, 0);

}

<style type="text/css">

h1 {

font-size: 34px;

}

h1.title {

font-size: 38px;

}

h2 {

font-size: 30px;

}

h3 {

font-size: 24px;

}

h4 {

font-size: 18px;

}

h5 {

font-size: 16px;

}

h6 {

font-size: 12px;

}

.table th:not([align]) {

text-align: left;

}

<style type = "text/css">

.main-container {

max-width: 940px;

margin-left: auto;

margin-right: auto;

}

code {

color: inherit;

background-color: rgba(0, 0, 0, 0.04);

}

img {

max-width:100%;

}

.tabbed-pane {

padding-top: 12px;

}

.html-widget {

margin-bottom: 20px;

}

button.code-folding-btn:focus {

outline: none;

}

summary {

display: list-item;

}

<style type="text/css">

body {

padding-top: 51px;

padding-bottom: 40px;

}

.section h1 {

padding-top: 56px;

margin-top: -56px;

}

.section h2 {

padding-top: 56px;

margin-top: -56px;

}

.section h3 {

padding-top: 56px;

margin-top: -56px;

}

.section h4 {

padding-top: 56px;

margin-top: -56px;

}

.section h5 {

padding-top: 56px;

margin-top: -56px;

}

.section h6 {

padding-top: 56px;

margin-top: -56px;

}

.dropdown-submenu {

position: relative;

}

.dropdown-submenu>.dropdown-menu {

top: 0;

left: 100%;

margin-top: -6px;

margin-left: -1px;

border-radius: 0 6px 6px 6px;

}

.dropdown-submenu:hover>.dropdown-menu {

display: block;

}

.dropdown-submenu>a:after {

display: block;

content: " ";

float: right;

width: 0;

height: 0;

border-color: transparent;

border-style: solid;

border-width: 5px 0 5px 5px;

border-left-color: #cccccc;

margin-top: 5px;

margin-right: -10px;

}

.dropdown-submenu:hover>a:after {

border-left-color: #ffffff;

}

.dropdown-submenu.pull-left {

float: none;

}

.dropdown-submenu.pull-left>.dropdown-menu {

left: -100%;

margin-left: 10px;

border-radius: 6px 0 6px 6px;

}

$(document).ready(function () {

// active menu anchor

href = window.location.pathname

href = href.substr(href.lastIndexOf('/') + 1)

if (href === "")

href = "index.html";

var menuAnchor = $('a[href="' + href + '"]');

// mark it active

menuAnchor.parent().addClass('active');

// if it's got a parent navbar menu mark it active as well

menuAnchor.closest('li.dropdown').addClass('active');

<style type="text/css">

.tabset-dropdown > .nav-tabs {

display: inline-table;

max-height: 500px;

min-height: 44px;

overflow-y: auto;

background: white;

border: 1px solid #ddd;

border-radius: 4px;

}

.tabset-dropdown > .nav-tabs > li.active:before {

content: "";

font-family: 'Glyphicons Halflings';

display: inline-block;

padding: 10px;

border-right: 1px solid #ddd;

}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li.active:before {

content: "&#xe258;";

border: none;

}

.tabset-dropdown > .nav-tabs.nav-tabs-open:before {

content: "";

font-family: 'Glyphicons Halflings';

display: inline-block;

padding: 10px;

border-right: 1px solid #ddd;

}

.tabset-dropdown > .nav-tabs > li.active {

display: block;

}

.tabset-dropdown > .nav-tabs > li > a,

.tabset-dropdown > .nav-tabs > li > a:focus,

.tabset-dropdown > .nav-tabs > li > a:hover {

border: none;

display: inline-block;

border-radius: 4px;

background-color: transparent;

}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li {

display: block;

float: none;

}

.tabset-dropdown > .nav-tabs > li {

display: none;

}

<style type="text/css">

#TOC {

margin: 25px 0px 20px 0px;

}

@media (max-width: 768px) {

#TOC {

position: relative;

width: 100%;

}

}

@media print {

.toc-content {

/* see https://github.com/w3c/csswg-drafts/issues/4434 */

float: right;

}

}

.toc-content {

padding-left: 30px;

padding-right: 40px;

}

div.main-container {

max-width: 1200px;

}

div.tocify {

width: 20%;

max-width: 260px;

max-height: 85%;

}

@media (min-width: 768px) and (max-width: 991px) {

div.tocify {

width: 25%;

}

}

@media (max-width: 767px) {

div.tocify {

width: 100%;

max-width: none;

}

}

.tocify ul, .tocify li {

line-height: 20px;

}

.tocify-subheader .tocify-item {

font-size: 0.90em;

}

.tocify .list-group-item {

border-radius: 0px;

}

<div class="container-fluid main-container">

<div class="row-fluid">

<div class="col-xs-12 col-sm-4 col-md-3">

<div id="TOC" class="tocify">

<div class="toc-content col-xs-12 col-sm-8 col-md-9">

<div class="navbar navbar-default navbar-fixed-top" role="navigation">

<div class="container">

<div class="navbar-header">

<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">

<span class="icon-bar"></span>

<span class="icon-bar"></span>

<span class="icon-bar"></span>

</button>

<a class="navbar-brand" href="index.html"></a>

</div>

<div id="navbar" class="navbar-collapse collapse">

<ul class="nav navbar-nav">

<li>

<a href="index.html">Intro</a>

<a href="Tutorial_1_Web_scraping.html">

<span class="ion ion-android-bulb"></span>

Tutorial 1

</a>

<a href="Tutorial_2_Read_textdata.html">

<span class="ion ion-android-bulb"></span>

Tutorial 2

</a>

<a href="Tutorial_3_Frequency.html">

<span class="ion ion-android-bulb"></span>

Tutorial 3

</a>

<a href="Tutorial_4_Term_extraction.html">

<span class="ion ion-android-bulb"></span>

Tutorial 4

</a>

<a href="Tutorial_5_Co-occurrence.html">

<span class="ion ion-android-bulb"></span>

Tutorial 5

</a>

<a href="Tutorial_6_Topic_Models.html">

<span class="ion ion-android-bulb"></span>

Tutorial 6

</a>

<a href="Tutorial_7_Klassifikation.html">

<span class="ion ion-android-bulb"></span>

Tutorial 7

</a>

<a href="Tutorial_8_NER_POS.html">

<span class="ion ion-android-bulb"></span>

Tutorial 8

</a>

</ul>

<ul class="nav navbar-nav navbar-right">

</ul>

</div><!--/.nav-collapse -->

</div><!--/.container -->

<div class="fluid-row" id="header">

<h1 class="title toc-ignore">Text mining in R for the social sciences and digital humanities</h1>

<h4 class="author">Andreas Niekler, Gregor Wiedemann</h4>

<h4 class="date">2020-10-08</h4>

<div id="contents" class="section level1">

<h1><span class="header-section-number">1</span> Contents</h1>

<p>In the digital age, large collections of text are an increasingly attractive data source for analysis in the social sciences. Corpora from thousands up to several millions of retro-digitized or natively digital documents cannot be investigated with conventional, manual methods alone. (Semi-)automatic computational analysis algorithms, also known as text mining, provide interesting opportunities for social scientists to extend their toolbox.</p>

<p>To realize complex designs in empirical social research, scientists need basic knowledge of computational algorithms to be able to select those appropriate for their needs. Specific projects may further require certain adaptations to standard procedures, language resources or analysis workflows. Instead of relying on off-the-shelf analysis software, using script programming languages is a very powerful way to fulfill such requirements. The course teaches an overview of text mining in connection with data acquisition, preprocessing and methodological integration using the statistical programming language R (www.r-project.org).</p>

<p>In sessions alternating between lectures and tutorials, we teach theoretical and methodological foundations, introduce exemplary studies and get hands on programming to realize different analyses.</p>

<div id="goals" class="section level1">

<h1><span class="header-section-number">2</span> Goals</h1>

<p>Participants will learn about opportunities and limits of text mining methods to analyze qualitative and quantitative aspects of large text collections. With example scripts provided in the programming language R, participants will learn how to realize single steps of such an analysis on a specific corpus. We cover a range of text mining methods from simple lexicometric measures such as word frequencies, key term extraction and co-occurrence analysis, to more complex machine learning approaches such as topic models and supervised text classification. The goal is to provide a broad overview of several technologies already established in social sciences. Participants will be enabled to identify their own priorities and to lay foundations for further independent studying tailored to their individual needs.</p>

<div id="requirements" class="section level1">

<h1><span class="header-section-number">3</span> Requirements</h1>

<p>We expect willingness to learn about algorithmic foundations of computational and statistical text analysis technologies. For the hands-on part, we rely on scripts in the programming language R. Thus, we strongly recommend some basic knowledge of R, to successfully take part in the tutorial sessions of the course.</p>

<p>If you already have a certain amount of knowledge in another programming language, learning R will be easy for you. However, since R is a statistical programming language, some of its concepts largely differ from other languages.</p>

<p>For participants without basic knowledge of R, we strongly recommend to learn at least a little in preparation of the course.</p>

<p>For a very brief overview of common R commands see: <a href="Tutorial_0_R-Intro.html">Tutorial_0_R-Intro.html</a></p>

<div id="tutorials" class="section level1">

<h1><span class="header-section-number">4</span> Tutorials</h1>

<p>The course consists of 8 tutorials written in R-markdown and further described in <a href="http://gscl2017.dfki.de/proceedings.php">this paper</a>.</p>

<p>You can use <em>knitr</em> to create the tutorial sheets as HTML notebooks from the <a href="https://github.com/tm4ss/tm4ss.github.io">R-markdown source code</a>.</p>

<ol style="list-style-type: decimal">

<li>Web crawling and scraping</li>

<li>Text data import in R</li>

<li>Frequency analysis</li>

<li>Key term extraction</li>

<li>Co-occurrence analysis</li>

<li>Topic models (LDA)</li>

<li>Text classification</li>

<li>Part-of-Speech tagging / Named Entity Recognition</li>

<div id="license" class="section level1">

<h1><span class="header-section-number">5</span> License</h1>

<p>This course was created by Gregor Wiedemann and Andreas Niekler. It was freely released under GPLv3 in September 2017. If you use (parts of) it for your own teaching or analysis, please cite</p>

<pre><code>Wiedemann, Gregor; Niekler, Andreas (2017): Hands-on: A five day text mining course for humanists and social scientists in R. Proceedings of the 1st Workshop on Teaching NLP for Digital Humanities (Teach4DH@GSCL 2017), Berlin.</code></pre>

<div id="session-info" class="section level1">

<h1><span class="header-section-number">6</span> Session info</h1>

<p>The tutorials load some of the following packages.</p>

<pre class="r"><code>package_list &lt;- c(

&quot;dplyr&quot;,

&quot;ggplot2&quot;,

&quot;igraph&quot;,

&quot;irr&quot;,

&quot;LDAvis&quot;,

&quot;LiblineaR&quot;,

&quot;Matrix&quot;,

&quot;NLP&quot;,

&quot;openNLP&quot;,

&quot;openNLPdata&quot;,

&quot;pals&quot;,

&quot;png&quot;,

&quot;quanteda&quot;,

&quot;readtext&quot;,

&quot;reshape2&quot;,

&quot;rvest&quot;,

&quot;topicmodels&quot;,

&quot;tsne&quot;,

&quot;webdriver&quot;,

&quot;wordcloud&quot;,

&quot;wordcloud2&quot;

)

lapply(package_list, require, character.only = TRUE)</code></pre>

<p>These are the package versions, we used for this tutorial.</p>

<pre class="r"><code>sessionInfo()</code></pre>

<pre><code>## R version 4.0.2 (2020-06-22)

## Platform: x86_64-w64-mingw32/x64 (64-bit)

## Running under: Windows 10 x64 (build 19041)

##

## Matrix products: default

##

## locale:

## [1] LC_COLLATE=German_Germany.1252 LC_CTYPE=German_Germany.1252

## [3] LC_MONETARY=German_Germany.1252 LC_NUMERIC=C

## [5] LC_TIME=German_Germany.1252

##

## attached base packages:

## [1] stats graphics grDevices utils datasets methods base

##

## other attached packages:

## [1] wordcloud2_0.2.1 wordcloud_2.6 RColorBrewer_1.1-2 webdriver_1.0.5

## [5] tsne_0.1-3 topicmodels_0.2-11 rvest_0.3.6 xml2_1.3.2

## [9] reshape2_1.4.4 readtext_0.76 quanteda_2.1.1 png_0.1-7

## [13] pals_1.6 openNLPdata_1.5.3-4 openNLP_0.2-7 NLP_0.2-0

## [17] Matrix_1.2-18 LiblineaR_2.10-8 LDAvis_0.3.2 irr_0.84.1

## [21] lpSolve_5.6.15 igraph_1.2.5 ggplot2_3.3.2 dplyr_1.0.2

##

## loaded via a namespace (and not attached):

## [1] httr_1.4.2 maps_3.3.0 jsonlite_1.7.1 RcppParallel_5.0.2

## [5] showimage_1.0.0 stats4_4.0.2 yaml_2.2.1 slam_0.1-47

## [9] pillar_1.4.6 lattice_0.20-41 glue_1.4.2 digest_0.6.25

## [13] colorspace_1.4-1 htmltools_0.5.0 plyr_1.8.6 tm_0.7-7

## [17] pkgconfig_2.0.3 purrr_0.3.4 scales_1.1.1 processx_3.4.4

## [21] tibble_3.0.3 generics_0.0.2 usethis_1.6.3 ellipsis_0.3.1

## [25] withr_2.2.0 magrittr_1.5 crayon_1.3.4 evaluate_0.14

## [29] ps_1.3.4 stopwords_2.0 fs_1.5.0 tools_4.0.2

## [33] data.table_1.13.0 lifecycle_0.2.0 stringr_1.4.0 munsell_0.5.0

## [37] callr_3.4.4 compiler_4.0.2 rlang_0.4.7 debugme_1.1.0

## [41] grid_4.0.2 dichromat_2.0-0 htmlwidgets_1.5.1 base64enc_0.1-3

## [45] rmarkdown_2.3 gtable_0.3.0 curl_4.3 R6_2.4.1

## [49] knitr_1.29 fastmatch_1.1-0 modeltools_0.2-23 rJava_0.9-13

## [53] stringi_1.5.3 parallel_4.0.2 Rcpp_1.0.5 vctrs_0.3.4

## [57] mapproj_1.2.7 tidyselect_1.1.0 xfun_0.17</code></pre>

<p>2020, Andreas Niekler and Gregor Wiedemann. GPLv3. <a href="https://tm4ss.github.io">tm4ss.github.io</a></p>

function bootstrapStylePandocTables() {

$('tr.header').parent('thead').parent('table').addClass('table table-condensed');

$(document).ready(function () {

bootstrapStylePandocTables();

$(document).ready(function () {

window.buildTabsets("TOC");

$(document).ready(function () {

$('.tabset-dropdown > .nav-tabs > li').click(function () {

$(this).parent().toggleClass('nav-tabs-open')

});

$(document).ready(function () {

// move toc-ignore selectors from section div to header

$('div.section.toc-ignore')

.removeClass('toc-ignore')

.children('h1,h2,h3,h4,h5').addClass('toc-ignore');

// establish options

var options = {

selectors: "h1,h2,h3",

theme: "bootstrap3",

context: '.toc-content',

hashGenerator: function (text) {

return text.replace(/[.\\/?&!#<>]/g, '').replace(/\s/g, '_');

},

ignoreSelector: ".toc-ignore",

scrollTo: 0

};

options.showAndHide = true;

options.smoothScroll = true;

// tocify

var toc = $("#TOC").tocify(options).data("toc-tocify");

(function () {

var script = document.createElement("script");

script.type = "text/javascript";

script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";

document.getElementsByTagName("head")[0].appendChild(script);

})();