<meta charset="utf-8" />

<meta name="generator" content="pandoc" />

<meta http-equiv="X-UA-Compatible" content="IE=EDGE" />

<meta name="author" content="Andreas Niekler, Gregor Wiedemann" />

<meta name="date" content="2020-10-08" />

<title>Tutorial 8: Part-of-Speech tagging / Named Entity Recognition</title>

<script src="site_libs/jquery-1.11.3/jquery.min.js"></script>

<meta name="viewport" content="width=device-width, initial-scale=1" />

<link href="site_libs/bootstrap-3.3.5/css/united.min.css" rel="stylesheet" />

<script src="site_libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>

<script src="site_libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>

<script src="site_libs/bootstrap-3.3.5/shim/respond.min.js"></script>

<script src="site_libs/jqueryui-1.11.4/jquery-ui.min.js"></script>

<link href="site_libs/tocify-1.9.1/jquery.tocify.css" rel="stylesheet" />

<script src="site_libs/tocify-1.9.1/jquery.tocify.js"></script>

<script src="site_libs/navigation-1.1/tabsets.js"></script>

<link href="site_libs/highlightjs-9.12.0/default.css" rel="stylesheet" />

<script src="site_libs/highlightjs-9.12.0/highlight.js"></script>

<script src="site_libs/clipboard-1.7.1/clipboard.min.js"></script>

<link href="site_libs/primer-tooltips-1.4.0/build.css" rel="stylesheet" />

<link href="site_libs/klippy-0.0.0.9500/css/klippy.min.css" rel="stylesheet" />

<script src="site_libs/klippy-0.0.0.9500/js/klippy.min.js"></script>

<link href="site_libs/ionicons-2.0.1/css/ionicons.min.css" rel="stylesheet" />

<style type="text/css">code{white-space: pre;}</style>

<style type="text/css">

pre:not([class]) {

background-color: white;

}

<script type="text/javascript">

if (window.hljs) {

hljs.configure({languages: []});

hljs.initHighlightingOnLoad();

if (document.readyState && document.readyState === "complete") {

window.setTimeout(function() { hljs.initHighlighting(); }, 0);

}

<style type="text/css">

h1 {

font-size: 34px;

}

h1.title {

font-size: 38px;

}

h2 {

font-size: 30px;

}

h3 {

font-size: 24px;

}

h4 {

font-size: 18px;

}

h5 {

font-size: 16px;

}

h6 {

font-size: 12px;

}

.table th:not([align]) {

text-align: left;

}

<style type = "text/css">

.main-container {

max-width: 940px;

margin-left: auto;

margin-right: auto;

}

code {

color: inherit;

background-color: rgba(0, 0, 0, 0.04);

}

img {

max-width:100%;

}

.tabbed-pane {

padding-top: 12px;

}

.html-widget {

margin-bottom: 20px;

}

button.code-folding-btn:focus {

outline: none;

}

summary {

display: list-item;

}

<style type="text/css">

body {

padding-top: 51px;

padding-bottom: 40px;

}

.section h1 {

padding-top: 56px;

margin-top: -56px;

}

.section h2 {

padding-top: 56px;

margin-top: -56px;

}

.section h3 {

padding-top: 56px;

margin-top: -56px;

}

.section h4 {

padding-top: 56px;

margin-top: -56px;

}

.section h5 {

padding-top: 56px;

margin-top: -56px;

}

.section h6 {

padding-top: 56px;

margin-top: -56px;

}

.dropdown-submenu {

position: relative;

}

.dropdown-submenu>.dropdown-menu {

top: 0;

left: 100%;

margin-top: -6px;

margin-left: -1px;

border-radius: 0 6px 6px 6px;

}

.dropdown-submenu:hover>.dropdown-menu {

display: block;

}

.dropdown-submenu>a:after {

display: block;

content: " ";

float: right;

width: 0;

height: 0;

border-color: transparent;

border-style: solid;

border-width: 5px 0 5px 5px;

border-left-color: #cccccc;

margin-top: 5px;

margin-right: -10px;

}

.dropdown-submenu:hover>a:after {

border-left-color: #ffffff;

}

.dropdown-submenu.pull-left {

float: none;

}

.dropdown-submenu.pull-left>.dropdown-menu {

left: -100%;

margin-left: 10px;

border-radius: 6px 0 6px 6px;

}

$(document).ready(function () {

// active menu anchor

href = window.location.pathname

href = href.substr(href.lastIndexOf('/') + 1)

if (href === "")

href = "index.html";

var menuAnchor = $('a[href="' + href + '"]');

// mark it active

menuAnchor.parent().addClass('active');

// if it's got a parent navbar menu mark it active as well

menuAnchor.closest('li.dropdown').addClass('active');

<style type="text/css">

.tabset-dropdown > .nav-tabs {

display: inline-table;

max-height: 500px;

min-height: 44px;

overflow-y: auto;

background: white;

border: 1px solid #ddd;

border-radius: 4px;

}

.tabset-dropdown > .nav-tabs > li.active:before {

content: "";

font-family: 'Glyphicons Halflings';

display: inline-block;

padding: 10px;

border-right: 1px solid #ddd;

}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li.active:before {

content: "&#xe258;";

border: none;

}

.tabset-dropdown > .nav-tabs.nav-tabs-open:before {

content: "";

font-family: 'Glyphicons Halflings';

display: inline-block;

padding: 10px;

border-right: 1px solid #ddd;

}

.tabset-dropdown > .nav-tabs > li.active {

display: block;

}

.tabset-dropdown > .nav-tabs > li > a,

.tabset-dropdown > .nav-tabs > li > a:focus,

.tabset-dropdown > .nav-tabs > li > a:hover {

border: none;

display: inline-block;

border-radius: 4px;

background-color: transparent;

}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li {

display: block;

float: none;

}

.tabset-dropdown > .nav-tabs > li {

display: none;

}

<style type="text/css">

#TOC {

margin: 25px 0px 20px 0px;

}

@media (max-width: 768px) {

#TOC {

position: relative;

width: 100%;

}

}

@media print {

.toc-content {

/* see https://github.com/w3c/csswg-drafts/issues/4434 */

float: right;

}

}

.toc-content {

padding-left: 30px;

padding-right: 40px;

}

div.main-container {

max-width: 1200px;

}

div.tocify {

width: 20%;

max-width: 260px;

max-height: 85%;

}

@media (min-width: 768px) and (max-width: 991px) {

div.tocify {

width: 25%;

}

}

@media (max-width: 767px) {

div.tocify {

width: 100%;

max-width: none;

}

}

.tocify ul, .tocify li {

line-height: 20px;

}

.tocify-subheader .tocify-item {

font-size: 0.90em;

}

.tocify .list-group-item {

border-radius: 0px;

}

<div class="container-fluid main-container">

<div class="row-fluid">

<div class="col-xs-12 col-sm-4 col-md-3">

<div id="TOC" class="tocify">

<div class="toc-content col-xs-12 col-sm-8 col-md-9">

<div class="navbar navbar-default navbar-fixed-top" role="navigation">

<div class="container">

<div class="navbar-header">

<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">

<span class="icon-bar"></span>

<span class="icon-bar"></span>

<span class="icon-bar"></span>

</button>

<a class="navbar-brand" href="index.html"></a>

</div>

<div id="navbar" class="navbar-collapse collapse">

<ul class="nav navbar-nav">

<li>

<a href="index.html">Intro</a>

<a href="Tutorial_1_Web_scraping.html">

<span class="ion ion-android-bulb"></span>

Tutorial 1

</a>

<a href="Tutorial_2_Read_textdata.html">

<span class="ion ion-android-bulb"></span>

Tutorial 2

</a>

<a href="Tutorial_3_Frequency.html">

<span class="ion ion-android-bulb"></span>

Tutorial 3

</a>

<a href="Tutorial_4_Term_extraction.html">

<span class="ion ion-android-bulb"></span>

Tutorial 4

</a>

<a href="Tutorial_5_Co-occurrence.html">

<span class="ion ion-android-bulb"></span>

Tutorial 5

</a>

<a href="Tutorial_6_Topic_Models.html">

<span class="ion ion-android-bulb"></span>

Tutorial 6

</a>

<a href="Tutorial_7_Klassifikation.html">

<span class="ion ion-android-bulb"></span>

Tutorial 7

</a>

<a href="Tutorial_8_NER_POS.html">

<span class="ion ion-android-bulb"></span>

Tutorial 8

</a>

</ul>

<ul class="nav navbar-nav navbar-right">

</ul>

</div><!--/.nav-collapse -->

</div><!--/.container -->

<div class="fluid-row" id="header">

<h1 class="title toc-ignore">Tutorial 8: Part-of-Speech tagging / Named Entity Recognition</h1>

<h4 class="author">Andreas Niekler, Gregor Wiedemann</h4>

<h4 class="date">2020-10-08</h4>

addClassKlippyTo("pre.r, pre.markdown");

addKlippy('left', 'top', 'auto', '1', 'Copy code', 'Copied!');

</script> This tutorial shows how to use Part-o-Speech-tagging (POS) with the openNLP package. The openNLP package relies on the <code>rjava</code> package. For this to work properly, you need a version of Java installed (e.g. open-jdk) which matches your R-version w.r.t either the 32- or 64-bit installation. Also the <code>JAVA_HOME</code> environment variable needs to be set, pointing to your Java installation directory.</p>

<div id="annotate-pos" class="section level1">

<h1><span class="header-section-number">1</span> Annotate POS</h1>

<p>We extract proper nouns (tag NNP for singular and tag NNPS for plural proper nouns) from paragraphs of president’s speeches.</p>

<pre class="r"><code>options(stringsAsFactors = FALSE)

library(quanteda)

library(NLP)

# read suto paragraphs

textdata &lt;- read.csv(&quot;data/sotu_paragraphs.csv&quot;, sep = &quot;;&quot;, encoding = &quot;UTF-8&quot;)

english_stopwords &lt;- readLines(&quot;resources/stopwords_en.txt&quot;, encoding = &quot;UTF-8&quot;)

# Create corpus object

sotu_corpus &lt;- corpus(textdata$text, docnames = textdata$doc_id)

require(openNLP)

require(openNLPdata)

# openNLP annotator objects

sent_token_annotator &lt;- openNLP::Maxent_Sent_Token_Annotator()

word_token_annotator &lt;- Maxent_Word_Token_Annotator()

pos_tag_annotator &lt;- Maxent_POS_Tag_Annotator()

annotator_pipeline &lt;- Annotator_Pipeline(

sent_token_annotator,

word_token_annotator,

pos_tag_annotator

)

# function for annotation

annotateDocuments &lt;- function(doc, pos_filter = NULL) {

doc &lt;- as.String(doc)

doc_with_annotations &lt;- NLP::annotate(doc, annotator_pipeline)

tags &lt;- sapply(subset(doc_with_annotations, type==&quot;word&quot;)$features, `[[`, &quot;POS&quot;)

tokens &lt;- doc[subset(doc_with_annotations, type==&quot;word&quot;)]

if (!is.null(pos_filter)) {

res &lt;- tokens[tags %in% pos_filter]

} else {

res &lt;- paste0(tokens, &quot;_&quot;, tags)

}

res &lt;- paste(res, collapse = &quot; &quot;)

return(res)

}

# run annotation on a sample of the corpus

annotated_corpus &lt;- lapply(texts(sotu_corpus)[1:10], annotateDocuments)

# Have a look into the first annotated documents

annotated_corpus[1]</code></pre>

<pre><code>## $`1`

## [1] &quot;Fellow-Citizens_NNS of_IN the_DT Senate_NNP and_CC House_NNP of_IN Representatives_NNPS :_:&quot;</code></pre>

<pre class="r"><code>annotated_corpus[2]</code></pre>

<pre><code>## $`2`

## [1] &quot;I_PRP embrace_VBP with_IN great_JJ satisfaction_NN the_DT opportunity_NN which_WDT now_RB presents_VBZ itself_PRP of_IN congratulating_VBG you_PRP on_IN the_DT present_JJ favorable_JJ prospects_NNS of_IN our_PRP$ public_JJ affairs_NNS ._. The_DT recent_JJ accession_NN of_IN the_DT important_JJ state_NN of_IN North_NNP Carolina_NNP to_TO the_DT Constitution_NNP of_IN the_DT United_NNP States_NNP (_-LRB- of_IN which_WDT official_JJ information_NN has_VBZ been_VBN received_VBN )_-RRB- ,_, the_DT rising_VBG credit_NN and_CC respectability_NN of_IN our_PRP$ country_NN ,_, the_DT general_JJ and_CC increasing_VBG good_JJ will_NN toward_IN the_DT government_NN of_IN the_DT Union_NNP ,_, and_CC the_DT concord_NN ,_, peace_NN ,_, and_CC plenty_NN with_IN which_WDT we_PRP are_VBP blessed_VBN are_VBP circumstances_NNS auspicious_JJ in_IN an_DT eminent_JJ degree_NN to_TO our_PRP$ national_JJ prosperity_NN ._.&quot;</code></pre>

<div id="filter-nes-for-further-applications" class="section level1">

<h1><span class="header-section-number">2</span> Filter NEs for further applications</h1>

<p>We annotate the first paragraphs of the corpus, extract proper nouns, also referred to as Named Entities (NEs) such as person names, locations etc., and compute significance of co-occurrence of them.</p>

<pre class="r"><code>sample_corpus &lt;- sapply(texts(sotu_corpus)[1:1000], annotateDocuments, pos_filter = c(&quot;NNP&quot;, &quot;NNPS&quot;, &quot;&quot;))

# Binary term matrix

require(Matrix)

minimumFrequency &lt;- 2

filtered_corpus &lt;- corpus(sample_corpus)

binDTM &lt;- filtered_corpus %&gt;%

tokens(what = &quot;fastestword&quot;) %&gt;%

tokens_tolower() %&gt;%

dfm() %&gt;%

dfm_weight(scheme = &quot;boolean&quot;)

# Matrix multiplication for cooccurrence counts

coocCounts &lt;- t(binDTM) %*% binDTM

source(&quot;calculateCoocStatistics.R&quot;)

# Definition of a parameter for the representation of the co-occurrences of a concept

# Determination of the term of which co-competitors are to be measured.

coocTerm &lt;- &quot;spain&quot;

coocs &lt;- calculateCoocStatistics(coocTerm, binDTM, measure=&quot;LOGLIK&quot;)

print(coocs[1:20])</code></pre>

<pre><code>## united states catholic spanish florida government majesty buenos

## 51.88 47.78 35.82 35.61 32.14 30.02 28.27 26.62

## ayres madrid south america february east pensacola state

## 26.62 21.52 21.52 12.23 11.63 9.54 8.10 7.69

## floridas france amelia st

## 6.96 5.78 5.37 4.54</code></pre>

<div id="german-language-support" class="section level1">

<h1><span class="header-section-number">3</span> German language support</h1>

<p>For German language support run</p>

<pre class="r"><code># install.packages(&quot;openNLPmodels.de&quot;, repos = &quot;http://datacube.wu.ac.at&quot;)

# require(&quot;openNLPmodels.de&quot;)</code></pre>

<div id="optional-exercises" class="section level1">

<h1><span class="header-section-number">4</span> Optional exercises</h1>

<ol style="list-style-type: decimal">

<li>Plot a co-occurrence graph for the term “family_NN” and its collocates, such as in tutorial 5.</li>

<li>Merging tokens by identical consecutive POS-tags can be a useful approach to identification of multi-word-units (MWU). Modify the function <code>annotateDocuments</code> in a way, that consecutive POS-tags get merged into a single token (e.g. “United_NNP States_NNP” becomes “United_States_NNP”).</li>

<li>Bring it all together: Create a topic model visualization (topic distribution per decade, Tutorial: Topic Models) based only on paragraphs related to Foreign Policy (Tutorial: Text Classification). Just use nouns (NN, NNS) and proper nouns (NNP, NNPS) for the model (Tutorial: POS-tagging).</li>

<p>2020, Andreas Niekler and Gregor Wiedemann. GPLv3. <a href="https://tm4ss.github.io">tm4ss.github.io</a></p>

function bootstrapStylePandocTables() {

$('tr.header').parent('thead').parent('table').addClass('table table-condensed');

$(document).ready(function () {

bootstrapStylePandocTables();

$(document).ready(function () {

window.buildTabsets("TOC");

$(document).ready(function () {

$('.tabset-dropdown > .nav-tabs > li').click(function () {

$(this).parent().toggleClass('nav-tabs-open')

});

$(document).ready(function () {

// move toc-ignore selectors from section div to header

$('div.section.toc-ignore')

.removeClass('toc-ignore')

.children('h1,h2,h3,h4,h5').addClass('toc-ignore');

// establish options

var options = {

selectors: "h1,h2,h3",

theme: "bootstrap3",

context: '.toc-content',

hashGenerator: function (text) {

return text.replace(/[.\\/?&!#<>]/g, '').replace(/\s/g, '_');

},

ignoreSelector: ".toc-ignore",

scrollTo: 0

};

options.showAndHide = true;

options.smoothScroll = true;

// tocify

var toc = $("#TOC").tocify(options).data("toc-tocify");

(function () {

var script = document.createElement("script");

script.type = "text/javascript";

script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";

document.getElementsByTagName("head")[0].appendChild(script);

})();