JoJo
/
LeOS-Genesis
mirror of https://github.com/LeOS-GSI/LeOS-Genesis
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bug Fixes 

Bug Fixes
master
Abdul Mannan Saeed 2022-05-02 09:26:09 -07:00
parent 70901fdcad
commit c3e5ddf540
781 changed files with 233250 additions and 135574 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/misc.xml
View File

@ -95,6 +95,7 @@
<entry key="..\:/workspace/Genesis-Android/app/src/main/res/layouts/home/layout/popup_side_menu.xml" value="1.0" />
<entry key="..\:/workspace/Genesis-Android/app/src/main/res/layouts/orbot/layout/orbot_settings_view.xml" value="0.18541666666666667" />
<entry key="..\:/workspace/Genesis-Android/app/src/main/res/layouts/proxyStatus/layout/proxy_status_view.xml" value="0.18541666666666667" />
<entry key="..\:/workspace/Genesis-Android/app/src/main/res/layouts/searchWidget/layout/widget_search_controller.xml" value="0.20989583333333334" />
<entry key="..\:/workspace/Genesis-Android/app/src/main/res/layouts/setting/layout/setting.xml" value="0.5" />
<entry key="..\:/workspace/Genesis-Android/app/src/main/res/layouts/setting/layout/setting_advance_view.xml" value="0.37083333333333335" />
<entry key="..\:/workspace/Genesis-Android/app/src/main/res/layouts/setting/layout/setting_advert_view.xml" value="0.33" />

6
app/build.gradle
View File

@ -85,10 +85,10 @@ dependencies {
implementation 'androidx.recyclerview:recyclerview:1.2.1'
implementation 'androidx.coordinatorlayout:coordinatorlayout:1.2.0'
implementation 'com.google.android.material:material:1.5.0'
implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.6.10'
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.10'
implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.6.21'
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.21'
implementation 'org.apache.commons:commons-text:1.3'
implementation 'androidx.work:work-runtime-ktx:2.8.0-alpha01'
implementation 'androidx.work:work-runtime-ktx:2.8.0-alpha02'
implementation 'androidx.constraintlayout:constraintlayout:2.1.3'
implementation 'com.android.support.constraint:constraint-layout:2.0.4'
implementation 'androidx.preference:preference-ktx:1.2.0'

48
app/src/main/assets/adblock/manifest.json
View File

@ -1,31 +1,33 @@
{
"description": "something something dark side",
"manifest_version": 2,
"name": "Interceptz",
"version": "1.0",
"icons": {
"description": "something something dark side",
"manifest_version": 2,
"name": "Interceptz",
"version": "1.0",
"icons": {
"48": "icons/border-48.png"
},
"applications": {
},
"applications": {
"gecko": {
"id": "borderify@mozilla.org",
"strict_min_version": "45.0"
"id": "borderify@mozilla.org",
"strict_min_version": "45.0"
}
},
"permissions": [
},
"permissions": [
"webRequest"
],
"background": {
"scripts": ["background.js"]
},
"content_scripts": [
],
"background": {
"scripts": [
"background.js"
]
},
"content_scripts": [
{
"matches": ["*://*.mozilla.org/*"],
"js": ["intercept.js"]
"matches": [
"*://*.mozilla.org/*"
],
"js": [
"intercept.js"
]
}
]
]
}

38
app/src/main/assets/error/badcert-dark.html
View File

File diff suppressed because one or more lines are too long

42
app/src/main/assets/error/badcert.html
View File

File diff suppressed because one or more lines are too long

40
app/src/main/assets/error/error-dark.html
View File

File diff suppressed because one or more lines are too long

44
app/src/main/assets/error/error.html
View File

File diff suppressed because one or more lines are too long

12
app/src/main/assets/geckoview-config-noimage.yaml
View File

@ -5,12 +5,12 @@ env:
prefs:
network.proxy.type: 1
network.proxy.type: 1
# network.proxy.socks: "127.0.0.1"
# network.proxy.socks_port: 9050
network.proxy.socks_version: 5
network.proxy.socks_remote_dns: true
permissions.default.image: 2
network.proxy.socks_version: 5
network.proxy.socks_remote_dns: true
permissions.default.image: 2
browser.display.show_image_placeholders: true
browser.cache.disk.enable: false
browser.cache.memory.enable: true
@ -18,10 +18,10 @@ prefs:
privacy.resistFingerprinting: true
privacy.donottrackheader.enabled: false
privacy.donottrackheader.value: 1
network.http.sendRefererHeader: 0
network.http.sendRefererHeader: 0
security.checkloaduri: false
security.mixed_content.block_active_content: false
security.mixed_content.block_display_content: false
security.mixed_content.block_display_content: false
media.peerconnection.enabled: false //webrtc disabled
browser.cache.disk_cache_ssl: true
signon.formlessCapture.enabled: true

12
app/src/main/assets/geckoview-config.yaml
View File

@ -5,12 +5,12 @@ env:
prefs:
network.proxy.type: 1
network.proxy.type: 1
# network.proxy.socks: "127.0.0.1"
# network.proxy.socks_port: 9050
network.proxy.socks_version: 5
network.proxy.socks_remote_dns: true
permissions.default.image: 1
network.proxy.socks_version: 5
network.proxy.socks_remote_dns: true
permissions.default.image: 1
browser.display.show_image_placeholders: true
browser.cache.disk.enable: false
browser.cache.memory.enable: true
@ -18,10 +18,10 @@ prefs:
privacy.resistFingerprinting: true
privacy.donottrackheader.enabled: false
privacy.donottrackheader.value: 1
network.http.sendRefererHeader: 0
network.http.sendRefererHeader: 0
security.checkloaduri: false
security.mixed_content.block_active_content: false
security.mixed_content.block_display_content: false
security.mixed_content.block_display_content: false
media.peerconnection.enabled: false //webrtc disabled
browser.cache.disk_cache_ssl: true
signon.formlessCapture.enabled: true

291
app/src/main/assets/help/help-dark.html
View File

@ -1,154 +1,179 @@
<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<!--Includes-->
<link rel="icon" href="../resources/images/favicon.ico">
<link rel="stylesheet" href="style/cs-help-dark.css" />
<link rel="stylesheet" href="../resources/style/cs-bootstrap.css">
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<!--Includes-->
<link rel="icon" href="../resources/images/favicon.ico">
<link rel="stylesheet" href="style/cs-help-dark.css"/>
<link rel="stylesheet" href="../resources/style/cs-bootstrap.css">
<!--Javascripts-->
<script src="../resources/javascript/jquery.js"></script>
<script src="../resources/javascript/bootstrap.bundle.min.js"></script>
<script type="../resources/javascript/bootstrap.js"></script>
<script type="../resources/javascript/bootstrap.min.js"></script>
<script type="text/javascript" src="javascript/js-help.js"></script>
<script src="../resources/javascript/jquery.min.js"></script>
<script src="../resources/javascript/bootstrap.min.js"></script>
<!--Javascripts-->
<script src="../resources/javascript/jquery.js"></script>
<script src="../resources/javascript/bootstrap.bundle.min.js"></script>
<script type="../resources/javascript/bootstrap.js"></script>
<script type="../resources/javascript/bootstrap.min.js"></script>
<script type="text/javascript" src="javascript/js-help.js"></script>
<script src="../resources/javascript/jquery.min.js"></script>
<script src="../resources/javascript/bootstrap.min.js"></script>
<!--Meta Data-->
<title>Orion Search Engine | Dark Web Based Search Engine</title>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Darkweb or Deepweb Search Engine working on multiple layered network including i2p,onion and riddle. Optimized Searches with mobile support and seperate Mobile Apps">
<!--Meta Data-->
<title>Orion Search Engine | Dark Web Based Search Engine</title>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description"
content="Darkweb or Deepweb Search Engine working on multiple layered network including i2p,onion and riddle. Optimized Searches with mobile support and seperate Mobile Apps">
</head>
</head>
<body>
<body>
<div class="container">
<p class="hi_info hi_no_select">The following<strong> help manual</strong> is loaded from local cache. Connect to proxy for viewing latest help repository</p>
<div id="accordion">
<div class="card hi_card">
<div class="card-header" data-toggle="collapse" href="#collapseZero">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 1 &nbsp;&nbsp; </span>Internet Connection Problem</strong>
</a>
</div>
<div id="collapseZero" class="collapse show" data-parent="#accordion">
<div class="card-body hi_bullets">
<li>Checking the network cables and router</li>
<li>Reconnect to Wi-Fi</li>
<li>Restart phone or application</li>
</div>
</div>
</div>
<div class="container">
<p class="hi_info hi_no_select">The following<strong> help manual</strong> is loaded from local
cache. Connect to proxy for viewing latest help repository</p>
<div id="accordion">
<div class="card hi_card">
<div class="card-header" data-toggle="collapse" href="#collapseZero">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 1 &nbsp;&nbsp; </span>Internet
Connection Problem</strong>
</a>
</div>
<div id="collapseZero" class="collapse show" data-parent="#accordion">
<div class="card-body hi_bullets">
<li>Checking the network cables and router</li>
<li>Reconnect to Wi-Fi</li>
<li>Restart phone or application</li>
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseOne">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 2 &nbsp;&nbsp; </span> View Connection Status </strong>
</a>
</div>
<div id="collapseOne" class="collapse" data-parent="#accordion">
<div class="card-body">
To see connection status with Tor network GOTO
<br><br>Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span> Settings <span class="hi_arrow--color">&#10132;</span> Onion Proxy Status
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseOne">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 2 &nbsp;&nbsp; </span>
View Connection Status </strong>
</a>
</div>
<div id="collapseOne" class="collapse" data-parent="#accordion">
<div class="card-body">
To see connection status with Tor network GOTO
<br><br>Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span>
Settings <span class="hi_arrow--color">&#10132;</span> Onion Proxy Status
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseTwo">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 3 &nbsp;&nbsp; </span> View Tor Logs </strong>
</a>
</div>
<div id="collapseTwo" class="collapse" data-parent="#accordion">
<div class="card-body">
You can view Tor logs to debug an issue. To do this GOTO
<br><br> Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span> Orion Logs
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseTwo">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 3 &nbsp;&nbsp; </span>
View Tor Logs </strong>
</a>
</div>
<div id="collapseTwo" class="collapse" data-parent="#accordion">
<div class="card-body">
You can view Tor logs to debug an issue. To do this GOTO
<br><br> Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span>
Orion Logs
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseThree">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 4 &nbsp;&nbsp; </span> Help manual not loading</strong>
</a>
</div>
<div id="collapseThree" class="collapse" data-parent="#accordion">
<div class="card-body">
This happens when we are unable to fetch help manual due to internet connection issue. It can also happen when ISP blocks some of our servers.
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseThree">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 4 &nbsp;&nbsp; </span>
Help manual not loading</strong>
</a>
</div>
<div id="collapseThree" class="collapse" data-parent="#accordion">
<div class="card-body">
This happens when we are unable to fetch help manual due to internet connection
issue. It can also happen when ISP blocks some of our servers.
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFour">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 5 &nbsp;&nbsp; </span> Tor routing blocked by ISP</strong>
</a>
</div>
<div id="collapseFour" class="collapse" data-parent="#accordion">
<div class="card-body">
Some times ISP blocks Tor network routing. This issue can be solved by enabling Tor Bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFour">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 5 &nbsp;&nbsp; </span>
Tor routing blocked by ISP</strong>
</a>
</div>
<div id="collapseFour" class="collapse" data-parent="#accordion">
<div class="card-body">
Some times ISP blocks Tor network routing. This issue can be solved by enabling
Tor Bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFive">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 6 &nbsp;&nbsp; </span> Orion not working in China</strong>
</a>
</div>
<div id="collapseFive" class="collapse" data-parent="#accordion">
<div class="card-body">
Tor routing can be blocked in China even when you are using default bridges. This issue can be solved by connecting with Meek bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span class="hi_arrow--color">&#10132;</span> Customize bridges <span class="hi_arrow--color">&#10132;</span> Meek-azure (China)
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFive">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 6 &nbsp;&nbsp; </span>
Orion not working in China</strong>
</a>
</div>
<div id="collapseFive" class="collapse" data-parent="#accordion">
<div class="card-body">
Tor routing can be blocked in China even when you are using default bridges.
This issue can be solved by connecting with Meek bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span
class="hi_arrow--color">&#10132;</span> Customize bridges <span
class="hi_arrow--color">&#10132;</span> Meek-azure (China)
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSix">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 7 &nbsp;&nbsp; </span> Provide a custom bridges</strong>
</a>
</div>
<div id="collapseSix" class="collapse" data-parent="#accordion">
<div class="card-body">
You can also provide custom bridges in Orion. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span class="hi_arrow--color">&#10132;</span> Customize bridges <span class="hi_arrow--color">&#10132;</span> Paste custom bridge <span class="hi_arrow--color">&#10132;</span> paste your bridge string
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSix">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 7 &nbsp;&nbsp; </span>
Provide a custom bridges</strong>
</a>
</div>
<div id="collapseSix" class="collapse" data-parent="#accordion">
<div class="card-body">
You can also provide custom bridges in Orion. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span
class="hi_arrow--color">&#10132;</span> Customize bridges <span
class="hi_arrow--color">&#10132;</span> Paste custom bridge <span
class="hi_arrow--color">&#10132;</span> paste your bridge string
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSeven">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 8 &nbsp;&nbsp; </span> Use Orion as a VPN </strong>
</a>
</div>
<div id="collapseSeven" class="collapse" data-parent="#accordion">
<div class="card-body">
Orion also provides a vpn from which you can secure data of other applications. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable VPN Service
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSeven">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 8 &nbsp;&nbsp; </span>
Use Orion as a VPN </strong>
</a>
</div>
<div id="collapseSeven" class="collapse" data-parent="#accordion">
<div class="card-body">
Orion also provides a vpn from which you can secure data of other applications.
To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable VPN Service
</div>
</div>
</div>
</div>
</div>
</body>
<br><br>
</div>
</div>
</body>
<br><br>
</html>

291
app/src/main/assets/help/help.html
View File

@ -1,154 +1,179 @@
<!DOCTYPE html>
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<!--Includes-->
<link rel="icon" href="../resources/images/favicon.ico">
<link rel="stylesheet" href="style/cs-help.css" />
<link rel="stylesheet" href="../resources/style/cs-bootstrap.css">
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<!--Includes-->
<link rel="icon" href="../resources/images/favicon.ico">
<link rel="stylesheet" href="style/cs-help.css"/>
<link rel="stylesheet" href="../resources/style/cs-bootstrap.css">
<!--Javascripts-->
<script src="../resources/javascript/jquery.js"></script>
<script src="../resources/javascript/bootstrap.bundle.min.js"></script>
<script type="../resources/javascript/bootstrap.js"></script>
<script type="../resources/javascript/bootstrap.min.js"></script>
<script type="text/javascript" src="javascript/js-help.js"></script>
<script src="../resources/javascript/jquery.min.js"></script>
<script src="../resources/javascript/bootstrap.min.js"></script>
<!--Javascripts-->
<script src="../resources/javascript/jquery.js"></script>
<script src="../resources/javascript/bootstrap.bundle.min.js"></script>
<script type="../resources/javascript/bootstrap.js"></script>
<script type="../resources/javascript/bootstrap.min.js"></script>
<script type="text/javascript" src="javascript/js-help.js"></script>
<script src="../resources/javascript/jquery.min.js"></script>
<script src="../resources/javascript/bootstrap.min.js"></script>
<!--Meta Data-->
<title>Orion Search Engine | Dark Web Based Search Engine</title>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Darkweb or Deepweb Search Engine working on multiple layered network including i2p,onion and riddle. Optimized Searches with mobile support and seperate Mobile Apps">
<!--Meta Data-->
<title>Orion Search Engine | Dark Web Based Search Engine</title>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description"
content="Darkweb or Deepweb Search Engine working on multiple layered network including i2p,onion and riddle. Optimized Searches with mobile support and seperate Mobile Apps">
</head>
</head>
<body>
<body>
<div class="container">
<p class="hi_info hi_no_select">The following<strong> help manual</strong> is loaded from local cache. Connect to proxy for viewing latest help repository</p>
<div id="accordion">
<div class="card hi_card">
<div class="card-header" data-toggle="collapse" href="#collapseZero">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 1 &nbsp;&nbsp; </span>Internet Connection Problem</strong>
</a>
</div>
<div id="collapseZero" class="collapse show" data-parent="#accordion">
<div class="card-body hi_bullets">
<li>Checking the network cables and router</li>
<li>Reconnect to Wi-Fi</li>
<li>Restart phone or application</li>
</div>
</div>
</div>
<div class="container">
<p class="hi_info hi_no_select">The following<strong> help manual</strong> is loaded from local
cache. Connect to proxy for viewing latest help repository</p>
<div id="accordion">
<div class="card hi_card">
<div class="card-header" data-toggle="collapse" href="#collapseZero">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 1 &nbsp;&nbsp; </span>Internet
Connection Problem</strong>
</a>
</div>
<div id="collapseZero" class="collapse show" data-parent="#accordion">
<div class="card-body hi_bullets">
<li>Checking the network cables and router</li>
<li>Reconnect to Wi-Fi</li>
<li>Restart phone or application</li>
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseOne">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 2 &nbsp;&nbsp; </span> View Connection Status </strong>
</a>
</div>
<div id="collapseOne" class="collapse" data-parent="#accordion">
<div class="card-body">
To see connection status with Tor network GOTO
<br><br>Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span> Settings <span class="hi_arrow--color">&#10132;</span> Onion Proxy Status
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseOne">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 2 &nbsp;&nbsp; </span>
View Connection Status </strong>
</a>
</div>
<div id="collapseOne" class="collapse" data-parent="#accordion">
<div class="card-body">
To see connection status with Tor network GOTO
<br><br>Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span>
Settings <span class="hi_arrow--color">&#10132;</span> Onion Proxy Status
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseTwo">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 3 &nbsp;&nbsp; </span> View Tor Logs </strong>
</a>
</div>
<div id="collapseTwo" class="collapse" data-parent="#accordion">
<div class="card-body">
You can view Tor logs to debug an issue. To do this GOTO
<br><br> Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span> Orion Logs
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseTwo">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 3 &nbsp;&nbsp; </span>
View Tor Logs </strong>
</a>
</div>
<div id="collapseTwo" class="collapse" data-parent="#accordion">
<div class="card-body">
You can view Tor logs to debug an issue. To do this GOTO
<br><br> Menu Icon (Top-Right) <span class="hi_arrow--color">&#10132;</span>
Orion Logs
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseThree">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 4 &nbsp;&nbsp; </span> Help manual not loading</strong>
</a>
</div>
<div id="collapseThree" class="collapse" data-parent="#accordion">
<div class="card-body">
This happens when we are unable to fetch help manual due to internet connection issue. It can also happen when ISP blocks some of our servers.
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseThree">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 4 &nbsp;&nbsp; </span>
Help manual not loading</strong>
</a>
</div>
<div id="collapseThree" class="collapse" data-parent="#accordion">
<div class="card-body">
This happens when we are unable to fetch help manual due to internet connection
issue. It can also happen when ISP blocks some of our servers.
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFour">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 5 &nbsp;&nbsp; </span> Tor routing blocked by ISP</strong>
</a>
</div>
<div id="collapseFour" class="collapse" data-parent="#accordion">
<div class="card-body">
Some times ISP blocks Tor network routing. This issue can be solved by enabling Tor Bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFour">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 5 &nbsp;&nbsp; </span>
Tor routing blocked by ISP</strong>
</a>
</div>
<div id="collapseFour" class="collapse" data-parent="#accordion">
<div class="card-body">
Some times ISP blocks Tor network routing. This issue can be solved by enabling
Tor Bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFive">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 6 &nbsp;&nbsp; </span> Orion not working in China</strong>
</a>
</div>
<div id="collapseFive" class="collapse" data-parent="#accordion">
<div class="card-body">
Tor routing can be blocked in China even when you are using default bridges. This issue can be solved by connecting with Meek bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span class="hi_arrow--color">&#10132;</span> Customize bridges <span class="hi_arrow--color">&#10132;</span> Meek-azure (China)
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseFive">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 6 &nbsp;&nbsp; </span>
Orion not working in China</strong>
</a>
</div>
<div id="collapseFive" class="collapse" data-parent="#accordion">
<div class="card-body">
Tor routing can be blocked in China even when you are using default bridges.
This issue can be solved by connecting with Meek bridges. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span
class="hi_arrow--color">&#10132;</span> Customize bridges <span
class="hi_arrow--color">&#10132;</span> Meek-azure (China)
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSix">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 7 &nbsp;&nbsp; </span> Provide a custom bridges</strong>
</a>
</div>
<div id="collapseSix" class="collapse" data-parent="#accordion">
<div class="card-body">
You can also provide custom bridges in Orion. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span class="hi_arrow--color">&#10132;</span> Customize bridges <span class="hi_arrow--color">&#10132;</span> Paste custom bridge <span class="hi_arrow--color">&#10132;</span> paste your bridge string
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSix">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 7 &nbsp;&nbsp; </span>
Provide a custom bridges</strong>
</a>
</div>
<div id="collapseSix" class="collapse" data-parent="#accordion">
<div class="card-body">
You can also provide custom bridges in Orion. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable Bridges <span
class="hi_arrow--color">&#10132;</span> Customize bridges <span
class="hi_arrow--color">&#10132;</span> Paste custom bridge <span
class="hi_arrow--color">&#10132;</span> paste your bridge string
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSeven">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 8 &nbsp;&nbsp; </span> Use Orion as a VPN </strong>
</a>
</div>
<div id="collapseSeven" class="collapse" data-parent="#accordion">
<div class="card-body">
Orion also provides a vpn from which you can secure data of other applications. To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear Icon <span class="hi_arrow--color">&#10132;</span> Enable VPN Service
</div>
</div>
</div>
<div class="card hi_card hi_card--margin-top">
<div class="card-header" data-toggle="collapse" href="#collapseSeven">
<a class="card-link">
<strong class="hi_card_link hi_no_select"><span class="hi_issue--color">Issue 8 &nbsp;&nbsp; </span>
Use Orion as a VPN </strong>
</a>
</div>
<div id="collapseSeven" class="collapse" data-parent="#accordion">
<div class="card-body">
Orion also provides a vpn from which you can secure data of other applications.
To do this GOTO
<br><br> Connection Screen <span class="hi_arrow--color">&#10132;</span> Gear
Icon <span class="hi_arrow--color">&#10132;</span> Enable VPN Service
</div>
</div>
</div>
</div>
</div>
</body>
<br><br>
</div>
</div>
</body>
<br><br>
</html>

6
app/src/main/assets/homepage/adn.svg
View File

@ -1 +1,5 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.0.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc. --><path d="M248 167.5l64.9 98.8H183.1l64.9-98.8zM496 256c0 136.9-111.1 248-248 248S0 392.9 0 256 111.1 8 248 8s248 111.1 248 248zm-99.8 82.7L248 115.5 99.8 338.7h30.4l33.6-51.7h168.6l33.6 51.7h30.2z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 496 512"><!--! Font Awesome Free 6.0.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc. -->
<path
d="M248 167.5l64.9 98.8H183.1l64.9-98.8zM496 256c0 136.9-111.1 248-248 248S0 392.9 0 256 111.1 8 248 8s248 111.1 248 248zm-99.8 82.7L248 115.5 99.8 338.7h30.4l33.6-51.7h168.6l33.6 51.7h30.2z" />
</svg>
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 478 B

After

Width:  |  Height:  |  Size: 497 B

293
app/src/main/assets/homepage/homepage-dark.html
View File

File diff suppressed because one or more lines are too long

289
app/src/main/assets/homepage/homepage.html
View File

File diff suppressed because one or more lines are too long

4
app/src/main/assets/parser/manifest.json
View File

@ -10,7 +10,9 @@
}
},
"externally_connectable": {
"matches": ["*"]
"matches": [
"*"
]
},
"content_scripts": [
{

333
app/src/main/assets/policy/policy.html
View File

@ -3,110 +3,263 @@
<head>
<!--Includes-->
<link rel="icon" href="../resources/images/favicon.ico">
<link rel="stylesheet" href="style/home/cs-privacy.css" />
<link rel="stylesheet" href="style/shared/cs-global-properties.css" />
<link rel="stylesheet" href="../resources/style/cs-bootstrap.css">
<link rel="stylesheet" href="style/shared/font-awesome/css/all.min.css"/>
<link rel="stylesheet" href="style/home/cs-homepage.css" />
<!--Includes-->
<link rel="icon" href="../resources/images/favicon.ico">
<link rel="stylesheet" href="style/home/cs-privacy.css"/>
<link rel="stylesheet" href="style/shared/cs-global-properties.css"/>
<link rel="stylesheet" href="../resources/style/cs-bootstrap.css">
<link rel="stylesheet" href="style/shared/font-awesome/css/all.min.css"/>
<link rel="stylesheet" href="style/home/cs-homepage.css"/>
<!--Javascripts-->
<script src="../resources/javascript/js-jquery.js"></script>
<script src="../resources/javascript/bootstrap.bundle.min.js"></script>
<script type="../resources/javascript/bootstrap.js"></script>
<script type="../resources/javascript/bootstrap.min.js"></script>
<script src="javascript/enum-homepage.js"></script>
<script src="javascript/string-homepage.js"></script>
<script src="javascript/js-homepage.js"></script>
<!--Javascripts-->
<script src="../resources/javascript/js-jquery.js"></script>
<script src="../resources/javascript/bootstrap.bundle.min.js"></script>
<script type="../resources/javascript/bootstrap.js"></script>
<script type="../resources/javascript/bootstrap.min.js"></script>
<script src="javascript/enum-homepage.js"></script>
<script src="javascript/string-homepage.js"></script>
<script src="javascript/js-homepage.js"></script>
<!--Meta Data-->
<title>Orion Search Engine | Dark Web Based Search Engine</title>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="Darkweb or Deepweb Search Engine working on multiple layered network including i2p,onion and riddle. Optimized Searches with mobile support and seperate Mobile Apps"> </head>
<!--Meta Data-->
<title>Orion Search Engine | Dark Web Based Search Engine</title>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description"
content="Darkweb or Deepweb Search Engine working on multiple layered network including i2p,onion and riddle. Optimized Searches with mobile support and seperate Mobile Apps">
</head>
<body class="clear_selection" id="pBody">
<body class="privacy">
<body class="clear_selection" id="pBody">
<body class="privacy">
<div class="privacy__welcome gs-disable-highlight">
<br>
<img class="privacy__icon gs-disable-highlight" src="images/privacy.png" alt="Loading...">
<p class="privacy__welcome-header gs-disable-highlight">We don't collect or share</p>
<p class="privacy__welcome-header privacy__welcome-header--bottom gs-disable-highlight">personal information.</p>
<div class="privacy__welcome gs-disable-highlight">
<br>
<img class="privacy__icon gs-disable-highlight" src="images/privacy.png" alt="Loading...">
<p class="privacy__welcome-header gs-disable-highlight">We don't collect or share</p>
<p class="privacy__welcome-header privacy__welcome-header--bottom gs-disable-highlight">personal
information.</p>
<p class="privacy__welcome-info gs-disable-highlight">That's our privacy policy in a nutshell.</p>
<a href="mailto:orionhiddentechnologies@gmail.com" class="gc--clear-selection" style="text-decoration: none"><div class="privacy__welcome-mail gs-disable-highlight">Contact Developer</div></a>
</div>
<div class="privacy__body">
<p class="privacy__welcome-info gs-disable-highlight">That's our privacy policy in a
nutshell.</p>
<a href="mailto:orionhiddentechnologies@gmail.com" class="gc--clear-selection"
style="text-decoration: none">
<div class="privacy__welcome-mail gs-disable-highlight">Contact Developer</div>
</a>
</div>
<div class="privacy__body">
<div class="privacy__paragraph-container privacy__paragraph-container--top">
<h4 class="privacy__body-header"><b>Search Leakage</b></h4>
<p class="privacy__body--info" >At other search engines, when you do a search and then click on a link, your search terms are sent to that site you clicked on (in the <a href="https://duckduckgo.com/?q=HTTP referrer header">HTTP referrer header</a>). We call this sharing of personal information "search leakage”, For example, when you search for something private, you are sharing that private search not only with your search engine, but also with all the sites that you clicked on (for that search). In addition, when you visit any site, your computer automatically sends information about it to that site (including your <a href="https://duckduckgo.com/?q=user agent">User agent</a> and <a href="https://duckduckgo.com/?q=IP address">IP address</a>). This information can often be used to identify you directly. So when you do that private search, not only can those other sites know your search terms, but they can also know that you searched it. It is this combination of available information about you that raises privacy concerns. Orion prevents search leakage by default. Instead, when you click on a link on our site, we route (redirect) that request in such a way so that it does not send your search terms to other sites. The other sites will still know that you visited them, but they will not know what search you entered beforehand. <br><br>At some other search engines (including us), you can also use an encrypted version (HTTPS), which as a byproduct doesn't usually send your search terms to sites. However, it is slower to connect to these versions and if you click on a site that also uses HTTPS then your search is sent. Nevertheless, the encrypted version does protect your search from being leaked onto the computers it travels on between you and us.</p>
<a class="anchor"></a>
<p class="privacy__body--info">At Orion, our encrypted version goes even further and automatically changes links from a number of major Web sites to point to the encrypted versions of those sites. It is modeled after (and uses code from) the HTTPS Everywhere. These sites include Wikipedia, Facebook, Twitter, and Amazon to name a few. Another way to prevent search leakage is by using something called a <a href="https://duckduckgo.com/?q=POST request">POST request</a>, which has the effect of not showing your search in your browser, and, as a consequence, does not send it to other sites. You can turn on POST requests on our settings page, but it has its own issues. POST requests usually break browser back buttons, and they make it impossible for you to easily share your search by copying and pasting it out of your Web browser's address bar. <br><br>Finally, if you want to prevent sites from knowing you visited them at all, you can use a proxy like <a href="https://duckduckgo.com/?q=Tor">Tor</a>. Orion actually operates a <a href="https://duckduckgo.com/?q=Tor exit enclave">Tor exit enclave</a>, which means you can get end to end anonymous and encrypted searching using Tor & DDG together. You can enter !proxy domain into Orion as well, and we will route you through a proxy, e.g. !proxy breadpig.com. Unfortunately, proxies can also be slow, and free proxies (like the one we use) are funded by arguably excessive advertising. Because of these drawbacks in HTTPS, POST and proxies we decided to take the redirect approach to combat search leakage. However, we leave the choice up to you. You can deviate from the default on our settings page by toggling the redirect or address bar settings. You can also use our encrypted version.</p>
<a class="anchor"></a>
</div>
<div class="privacy__paragraph-container privacy__paragraph-container--top">
<h4 class="privacy__body-header"><b>Search Leakage</b></h4>
<p class="privacy__body--info">At other search engines, when you do a search and then click
on a link, your search terms are sent to that site you clicked on (in the <a
href="https://duckduckgo.com/?q=HTTP referrer header">HTTP referrer header</a>).
We call this sharing of personal information "search leakage”, For example, when you
search for something private, you are sharing that private search not only with your
search engine, but also with all the sites that you clicked on (for that search). In
addition, when you visit any site, your computer automatically sends information about
it to that site (including your <a href="https://duckduckgo.com/?q=user agent">User
agent</a> and <a href="https://duckduckgo.com/?q=IP address">IP address</a>). This
information can often be used to identify you directly. So when you do that private
search, not only can those other sites know your search terms, but they can also know
that you searched it. It is this combination of available information about you that
raises privacy concerns. Orion prevents search leakage by default. Instead, when you
click on a link on our site, we route (redirect) that request in such a way so that it
does not send your search terms to other sites. The other sites will still know that you
visited them, but they will not know what search you entered beforehand. <br><br>At some
other search engines (including us), you can also use an encrypted version (HTTPS),
which as a byproduct doesn't usually send your search terms to sites. However, it is
slower to connect to these versions and if you click on a site that also uses HTTPS then
your search is sent. Nevertheless, the encrypted version does protect your search from
being leaked onto the computers it travels on between you and us.</p>
<a class="anchor"></a>
<p class="privacy__body--info">At Orion, our encrypted version goes even further and
automatically changes links from a number of major Web sites to point to the encrypted
versions of those sites. It is modeled after (and uses code from) the HTTPS Everywhere.
These sites include Wikipedia, Facebook, Twitter, and Amazon to name a few. Another way
to prevent search leakage is by using something called a <a
href="https://duckduckgo.com/?q=POST request">POST request</a>, which has the
effect of not showing your search in your browser, and, as a consequence, does not send
it to other sites. You can turn on POST requests on our settings page, but it has its
own issues. POST requests usually break browser back buttons, and they make it
impossible for you to easily share your search by copying and pasting it out of your Web
browser's address bar. <br><br>Finally, if you want to prevent sites from knowing you
visited them at all, you can use a proxy like <a href="https://duckduckgo.com/?q=Tor">Tor</a>.
Orion actually operates a <a href="https://duckduckgo.com/?q=Tor exit enclave">Tor exit
enclave</a>, which means you can get end to end anonymous and encrypted searching
using Tor & DDG together. You can enter !proxy domain into Orion as well, and we will
route you through a proxy, e.g. !proxy breadpig.com. Unfortunately, proxies can also be
slow, and free proxies (like the one we use) are funded by arguably excessive
advertising. Because of these drawbacks in HTTPS, POST and proxies we decided to take
the redirect approach to combat search leakage. However, we leave the choice up to you.
You can deviate from the default on our settings page by toggling the redirect or
address bar settings. You can also use our encrypted version.</p>
<a class="anchor"></a>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Search History</b></h4>
<p class="privacy__body--info">Other search engines save your search history. Usually your searches are saved along with the date and time of the search, some information about your computer (e.g. your IP address, User agent and often a unique identifier stored in a <a href="https://duckduckgo.com/?q=browser cookie">browser cookie</a>), and if you are logged in, your account information (e.g. name and email address). With only the timestamp and computer information, your searches can often be traced directly to you. With the additional account information, they are associated directly with you. Also, note that with this information your searches can be tied together. This means someone can see everything you've been searching, not just one isolated search. You can usually find out a lot about a person from their search history. It's sort of creepy that people at search engines can see all this info about you, but that is not the main concern. The main concern is when they either a) release it to the public or b) give it to law enforcement. Why would they release it to the public? AOL famously released supposedly anonymous search terms for research purposes, except they didn't do a good job of making them completely anonymous, and they were ultimately. <br><br>The other way to release it to the public is by accident. Search engines could lose data, or get hacked, or accidentally expose data due to security holes or incompetence, all of which has happened with personal information on the Internet. Why would search engines give your search history to law enforcement? Simply because law enforcement asked for it, usually as part of a legal investigation. If you read privacy policies and terms of service carefully you will notice that they say they can give your information on court order. This makes sense because they may be legally obligated to do so. However, search engines are not legally obligated to collect personal information in the first place. They do it on their own volition. The bottom line is if search engines have your information, it could get out, even if they have the best intentions. And this information (your search history) can be pretty personal. <br><br>For these reasons, Orion takes the approach to not collect any personal information. The decisions of whether and how to comply with law enforcement requests, whether and how to anonymize data, and how to best protect your information from hackers are out of our hands. Your search history is safe with us because it cannot be tied to you in any way.</p>
<a class="anchor"></a>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Search History</b></h4>
<p class="privacy__body--info">Other search engines save your search history. Usually your
searches are saved along with the date and time of the search, some information about
your computer (e.g. your IP address, User agent and often a unique identifier stored in
a <a href="https://duckduckgo.com/?q=browser cookie">browser cookie</a>), and if you are
logged in, your account information (e.g. name and email address). With only the
timestamp and computer information, your searches can often be traced directly to you.
With the additional account information, they are associated directly with you. Also,
note that with this information your searches can be tied together. This means someone
can see everything you've been searching, not just one isolated search. You can usually
find out a lot about a person from their search history. It's sort of creepy that people
at search engines can see all this info about you, but that is not the main concern. The
main concern is when they either a) release it to the public or b) give it to law
enforcement. Why would they release it to the public? AOL famously released supposedly
anonymous search terms for research purposes, except they didn't do a good job of making
them completely anonymous, and they were ultimately. <br><br>The other way to release it
to the public is by accident. Search engines could lose data, or get hacked, or
accidentally expose data due to security holes or incompetence, all of which has
happened with personal information on the Internet. Why would search engines give your
search history to law enforcement? Simply because law enforcement asked for it, usually
as part of a legal investigation. If you read privacy policies and terms of service
carefully you will notice that they say they can give your information on court order.
This makes sense because they may be legally obligated to do so. However, search engines
are not legally obligated to collect personal information in the first place. They do it
on their own volition. The bottom line is if search engines have your information, it
could get out, even if they have the best intentions. And this information (your search
history) can be pretty personal. <br><br>For these reasons, Orion takes the approach to
not collect any personal information. The decisions of whether and how to comply with
law enforcement requests, whether and how to anonymize data, and how to best protect
your information from hackers are out of our hands. Your search history is safe with us
because it cannot be tied to you in any way.</p>
<a class="anchor"></a>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Information Not Collected</b></h4>
<p class="privacy__body--info">When you search at Orion, we don't know who you are and there is no way to tie your searches together. When you access Orion (or any Web site), your Web browser automatically sends information about your computer, e.g. your User agent and IP address because this information could be used to link you to your searches, we do not log (store) it at all. This is a very unusual practice, but we feel it is an important step to protect your privacy. It is unusual for a few reasons. First, most server software auto-stores this information, so you have to go out of your way not to store it. Second, most businesses want to keep as much information as possible because they don't know when it will be useful. Third, many search engines actively use this information, for example to show you more targeted advertising. <br><br>Another way that your searches are often tied together at other search engines are through browser cookies, which are pieces of information that sit on your computer and get sent to the search engine on each request. What search engines often do is store a unique identifier in your browser and then associate that identifier with your searches. <br><br>At Orion, no cookies are used by default. In response to efforts by the EFF and others, the major search engines have begun "anonymizing" their search log data after periods of time. Sure, this is better than not doing so, but you should note that this does not make your search history anonymous in the same way that it is at Orion. What search engines generally do when they anonymize data is get rid of part of your IP address or turn it into something that doesn't look exactly like an IP address and they do the same thing for uniquely identifying cookies. However, in many cases, this so-called anonymous data can still tie your searches together, which can be used to reconstruct who you are and what you searched for. Additionally, search engines usually are silent on what they do with the User agent, which has been shown to also have enough information to often be personally identifiable, especially if isolated to a particular search session (day).</p>
<a class="anchor"></a>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Information Not Collected</b></h4>
<p class="privacy__body--info">When you search at Orion, we don't know who you are and there
is no way to tie your searches together. When you access Orion (or any Web site), your
Web browser automatically sends information about your computer, e.g. your User agent
and IP address because this information could be used to link you to your searches, we
do not log (store) it at all. This is a very unusual practice, but we feel it is an
important step to protect your privacy. It is unusual for a few reasons. First, most
server software auto-stores this information, so you have to go out of your way not to
store it. Second, most businesses want to keep as much information as possible because
they don't know when it will be useful. Third, many search engines actively use this
information, for example to show you more targeted advertising. <br><br>Another way that
your searches are often tied together at other search engines are through browser
cookies, which are pieces of information that sit on your computer and get sent to the
search engine on each request. What search engines often do is store a unique identifier
in your browser and then associate that identifier with your searches. <br><br>At Orion,
no cookies are used by default. In response to efforts by the EFF and others, the major
search engines have begun "anonymizing" their search log data after periods of time.
Sure, this is better than not doing so, but you should note that this does not make your
search history anonymous in the same way that it is at Orion. What search engines
generally do when they anonymize data is get rid of part of your IP address or turn it
into something that doesn't look exactly like an IP address and they do the same thing
for uniquely identifying cookies. However, in many cases, this so-called anonymous data
can still tie your searches together, which can be used to reconstruct who you are and
what you searched for. Additionally, search engines usually are silent on what they do
with the User agent, which has been shown to also have enough information to often be
personally identifiable, especially if isolated to a particular search session (day).
</p>
<a class="anchor"></a>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Information Collected</b></h4>
<p class="privacy__body--info">At Orion, no cookies are used by default. If you have changed any settings, then cookies are used to store those changes. However, in that case, they are not stored in a personally identifiable way. For example, the large size setting is stored as 's=l'; no unique identifier is in there. Furthermore, if you prefer not to use cookies to store settings, you can use URL parameters instead.
<br><br>We do not use any third parties to do the code insertion, and we do not work with any sites that share personally identifiable information (e.g. name, address, etc.) via their affiliate programs. This means that no information is shared from Orion to the sites, and the only information that is collected from this process is product information, which is not tied to any particular user and which we do not save or store on our end. It is completely analogous to the search result case from the previous paragraph--we can see anonymous product info such that we cannot tie them to any particular person (or even tie multiple purchases together). <br><br>This whole affiliate process is an attempt to keep advertising to a minimal level on Orion. Finally, if you give us feedback, it may be stored in our email. However, you can give anonymous feedback (by not entering your email or other personal info on the feedback form).
</p>
<a class="anchor"></a>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Information Collected</b></h4>
<p class="privacy__body--info">At Orion, no cookies are used by default. If you have changed
any settings, then cookies are used to store those changes. However, in that case, they
are not stored in a personally identifiable way. For example, the large size setting is
stored as 's=l'; no unique identifier is in there. Furthermore, if you prefer not to use
cookies to store settings, you can use URL parameters instead.
<br><br>We do not use any third parties to do the code insertion, and we do not work
with any sites that share personally identifiable information (e.g. name, address, etc.)
via their affiliate programs. This means that no information is shared from Orion to the
sites, and the only information that is collected from this process is product
information, which is not tied to any particular user and which we do not save or store
on our end. It is completely analogous to the search result case from the previous
paragraph--we can see anonymous product info such that we cannot tie them to any
particular person (or even tie multiple purchases together). <br><br>This whole
affiliate process is an attempt to keep advertising to a minimal level on Orion.
Finally, if you give us feedback, it may be stored in our email. However, you can give
anonymous feedback (by not entering your email or other personal info on the feedback
form).
</p>
<a class="anchor"></a>
</div>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Information Shared</b></h4>
<p class="privacy__body--info">If you turn redirects off in the settings and you don't either turn POST on or use our encrypted site, then your search could leak to sites you click on. Yet as explained above, this does not happen by default. Also, like anyone else, we will comply with court ordered legal requests. However, in our case, we don't expect any because there is nothing useful to give them since we don't collect any personal information</p>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Information Shared</b></h4>
<p class="privacy__body--info">If you turn redirects off in the settings and you don't
either turn POST on or use our encrypted site, then your search could leak to sites you
click on. Yet as explained above, this does not happen by default. Also, like anyone
else, we will comply with court ordered legal requests. However, in our case, we don't
expect any because there is nothing useful to give them since we don't collect any
personal information</p>
</div>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Web Content Policy</b></h4>
<p class="privacy__body--info"> This SERVICE is provided by at no cost and is intended for use as is. This page is used to inform visitors regarding my policies with the collection, use, and disclosure of Personal Information if anyone decided to use my Service. If you choose to use my Service, then you agree to the collection and use of information in relation to this policy. The Personal Information that we collect is used for providing and improving the Service. We will not use or share your information with anyone except as described in this Privacy Policy. The terms used in this Privacy Policy have the same meanings as in our Terms and Conditions, which is accessible at <a href="https://boogle.store/privacy">https://boogle.store/privacy</a> unless otherwise defined in this Privacy Policy. </p>
<hr class="hr privacy__line-break"/>
<div class="privacy__paragraph-container">
<h4 class="privacy__body-header"><b>Web Content Policy</b></h4>
<p class="privacy__body--info"> This SERVICE is provided by at no cost and is intended for
use as is. This page is used to inform visitors regarding my policies with the
collection, use, and disclosure of Personal Information if anyone decided to use my
Service. If you choose to use my Service, then you agree to the collection and use of
information in relation to this policy. The Personal Information that we collect is used
for providing and improving the Service. We will not use or share your information with
anyone except as described in this Privacy Policy. The terms used in this Privacy Policy
have the same meanings as in our Terms and Conditions, which is accessible at <a
href="https://boogle.store/privacy">https://boogle.store/privacy</a> unless
otherwise defined in this Privacy Policy. </p>
<p class="privacy__body--info">
For a better experience, while using our Service, we may collect a certain personally identifiable information that is Advertisement Identifier. The information that we collect is retained on your device and is not collected by me in any way. The app does use third party services that may collect information used to identify you. Link to privacy policy of third party service providers used by the app are <br>
<ul class="privacy__body-list">
<li>Google Play Services: <a href="https://policies.google.com/privacy">https://policies.google.com/privacy</a></li>
<li><a href="https://support.google.com/admob/answer/6128543?hl=en">https://support.google.com/admob/answer/6128543?hl=en</a></li>
</ul>
<p class="privacy__body--info">
This application collects user android advertisement identifier and sends it to third party that is google admob for displaying best advertisement suited for that individual but We don’t personally maintain or use this advertisement identifier in any way. We may employ third-party companies and individuals due to the following reasons:
</p>
<p class="privacy__body--info">
<ul class="privacy__body-list">
<li>To facilitate our Service</li>
<li>To provide the Service on our behalf</li>
<li>To perform Service-related services or</li>
<li>To analyzing service usage</li>
</ul>
<p class="privacy__body--info">We want to inform users of this Service that these third parties have access to your Personal Information. The reason is to perform the tasks assigned to them on our behalf. However, they are obligated not to disclose or use the information for any other purpose. We value your trust in providing us your Personal Information, thus we are striving to use commercially acceptable means of protecting it. But remember that no method of transmission over the internet, or method of electronic storage is 100% secure and reliable, and we cannot guarantee its absolute security</p>
<p class="privacy__body--info">This Service only contain links to other sites. If you click on a third-party link, you will be directed to that site. Note that these external sites are not operated by us. Therefore, we strongly advise you to review the Privacy Policy of these websites. We have no control over and assume no responsibility for the content, privacy policies, or practices of any third-party sites or services. We may update our Privacy Policy from time to time. Thus, you are advised to review this page periodically for any changes. We will notify you of any changes by posting the new Privacy Policy on this page. These changes are effective immediately after they are posted on this page.</p><br><br><br>
</div>
</div>
<p class="privacy__body--info">
For a better experience, while using our Service, we may collect a certain personally
identifiable information that is Advertisement Identifier. The information that we
collect is retained on your device and is not collected by me in any way. The app does
use third party services that may collect information used to identify you. Link to
privacy policy of third party service providers used by the app are <br>
<ul class="privacy__body-list">
<li>Google Play Services: <a href="https://policies.google.com/privacy">https://policies.google.com/privacy</a>
</li>
<li><a href="https://support.google.com/admob/answer/6128543?hl=en">https://support.google.com/admob/answer/6128543?hl=en</a>
</li>
</ul>
<p class="privacy__body--info">
This application collects user android advertisement identifier and sends it to third
party that is google admob for displaying best advertisement suited for that individual
but We don’t personally maintain or use this advertisement identifier in any way. We may
employ third-party companies and individuals due to the following reasons:
</p>
<p class="privacy__body--info">
<ul class="privacy__body-list">
<li>To facilitate our Service</li>
<li>To provide the Service on our behalf</li>
<li>To perform Service-related services or</li>
<li>To analyzing service usage</li>
</ul>
<p class="privacy__body--info">We want to inform users of this Service that these third
parties have access to your Personal Information. The reason is to perform the tasks
assigned to them on our behalf. However, they are obligated not to disclose or use the
information for any other purpose. We value your trust in providing us your Personal
Information, thus we are striving to use commercially acceptable means of protecting it.
But remember that no method of transmission over the internet, or method of electronic
storage is 100% secure and reliable, and we cannot guarantee its absolute security</p>
<p class="privacy__body--info">This Service only contain links to other sites. If you click
on a third-party link, you will be directed to that site. Note that these external sites
are not operated by us. Therefore, we strongly advise you to review the Privacy Policy
of these websites. We have no control over and assume no responsibility for the content,
privacy policies, or practices of any third-party sites or services. We may update our
Privacy Policy from time to time. Thus, you are advised to review this page periodically
for any changes. We will notify you of any changes by posting the new Privacy Policy on
this page. These changes are effective immediately after they are posted on this
page.</p><br><br><br>
</div>
</div>
</body>
</body>
</body>
</body>
</html>

2
app/src/main/assets/readability/.taskcluster.yml
View File

@ -18,7 +18,7 @@ tasks:
$eval: as_slugid("pr_task")
provisionerId: proj-misc
workerType: ci
deadline: {$fromNow: '1 day'}
deadline: { $fromNow: '1 day' }
payload:
maxRunTime: 600
image: node

63
app/src/main/assets/readability/test/test-pages/001/expected.html
View File

@ -1,22 +1,41 @@
<div id="readability-page-1" class="page">
<section>
<p><strong>So finally you're <a href="http://fakehost/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">testing your frontend JavaScript code</a>? Great! The more you write tests, the more confident you are with your code… but how much precisely? That's where <a href="http://en.wikipedia.org/wiki/Code_coverage">code coverage</a> might help.</strong>
<p><strong>So finally you're <a
href="http://fakehost/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">testing
your frontend JavaScript code</a>? Great! The more you write tests, the more confident
you are with your code… but how much precisely? That's where <a
href="http://en.wikipedia.org/wiki/Code_coverage">code coverage</a> might help.</strong>
</p>
<p>The idea behind code coverage is to record which parts of your code (functions, statements, conditionals and so on) have been executed by your test suite, to compute metrics out of these data and usually to provide tools for navigating and inspecting them.</p>
<p>Not a lot of frontend developers I know actually test their frontend code, and I can barely imagine how many of them have ever setup code coverage… Mostly because there are not many frontend-oriented tools in this area I guess.</p>
<p>Actually I've only found one which provides an adapter for <a href="http://visionmedia.github.io/mocha/">Mocha</a> and actually works…</p>
<p>The idea behind code coverage is to record which parts of your code (functions,
statements, conditionals and so on) have been executed by your test suite, to compute
metrics out of these data and usually to provide tools for navigating and inspecting
them.</p>
<p>Not a lot of frontend developers I know actually test their frontend code, and I can
barely imagine how many of them have ever setup code coverage… Mostly because there are
not many frontend-oriented tools in this area I guess.</p>
<p>Actually I've only found one which provides an adapter for <a
href="http://visionmedia.github.io/mocha/">Mocha</a> and actually works…</p>
<blockquote>
<p>Drinking game for web devs: <br />(1) Think of a noun <br />(2) Google "&lt;noun&gt;.js" <br />(3) If a library with that name exists - drink</p>— Shay Friedman (@ironshay) <a href="https://twitter.com/ironshay/statuses/370525864523743232">August 22, 2013</a>
<p>Drinking game for web devs: <br/>(1) Think of a noun <br/>(2) Google "&lt;noun&gt;.js"
<br/>(3) If a library with that name exists - drink</p>— Shay Friedman (@ironshay)
<a href="https://twitter.com/ironshay/statuses/370525864523743232">August 22, 2013</a>
</blockquote>
<p><strong><a href="http://blanketjs.org/">Blanket.js</a></strong> is an <em>easy to install, easy to configure, and easy to use JavaScript code coverage library that works both in-browser and with nodejs.</em>
<p><strong><a href="http://blanketjs.org/">Blanket.js</a></strong> is an <em>easy to
install, easy to configure, and easy to use JavaScript code coverage library that works
both in-browser and with nodejs.</em>
</p>
<p>Its use is dead easy, adding Blanket support to your Mocha test suite is just matter of adding this simple line to your HTML test file:</p>
<p>Its use is dead easy, adding Blanket support to your Mocha test suite is just matter of
adding this simple line to your HTML test file:</p>
<pre><code>&lt;script src="vendor/blanket.js"
data-cover-adapter="vendor/mocha-blanket.js"&gt;&lt;/script&gt;
</code></pre>
<p>Source files: <a href="https://raw.github.com/alex-seville/blanket/master/dist/qunit/blanket.min.js">blanket.js</a>, <a href="https://raw.github.com/alex-seville/blanket/master/src/adapters/mocha-blanket.js">mocha-blanket.js</a>
<p>Source files: <a
href="https://raw.github.com/alex-seville/blanket/master/dist/qunit/blanket.min.js">blanket.js</a>,
<a href="https://raw.github.com/alex-seville/blanket/master/src/adapters/mocha-blanket.js">mocha-blanket.js</a>
</p>
<p>As an example, let's reuse the silly <code>Cow</code> example we used <a href="http://fakehost/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">in a previous episode</a>:</p>
<p>As an example, let's reuse the silly <code>Cow</code> example we used <a
href="http://fakehost/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">in
a previous episode</a>:</p>
<pre><code>// cow.js
(function(exports) {
"use strict";
@ -84,16 +103,30 @@ describe("Cow", function() {
</code></pre>
<p><strong>Notes</strong>:</p>
<ul>
<li>Notice the <code>data-cover</code> attribute we added to the script tag loading the source of our library;</li>
<li>The HTML test file <em>must</em> be served over HTTP for the adapter to be loaded.</li>
<li>Notice the <code>data-cover</code> attribute we added to the script tag loading the
source of our library;
</li>
<li>The HTML test file <em>must</em> be served over HTTP for the adapter to be loaded.
</li>
</ul>
<p>Running the tests now gives us something like this:</p>
<p>
<img alt="screenshot" src="http://fakehost/static/code/2013/blanket-coverage.png" />
<img alt="screenshot" src="http://fakehost/static/code/2013/blanket-coverage.png"/>
</p>
<p>As you can see, the report at the bottom highlights that we haven't actually tested the case where an error is raised in case a target name is missing. We've been informed of that, nothing more, nothing less. We simply know we're missing a test here. Isn't this cool? I think so!</p>
<p>Just remember that code coverage will only <a href="http://codebetter.com/karlseguin/2008/12/09/code-coverage-use-it-wisely/">bring you numbers</a> and raw information, not actual proofs that the whole of your <em>code logic</em> has been actually covered. If you ask me, the best inputs you can get about your code logic and implementation ever are the ones issued out of <a href="http://www.extremeprogramming.org/rules/pair.html">pair programming</a> sessions and <a href="http://alexgaynor.net/2013/sep/26/effective-code-review/">code reviews</a> — but that's another story.</p>
<p><strong>So is code coverage silver bullet? No. Is it useful? Definitely. Happy testing!</strong>
<p>As you can see, the report at the bottom highlights that we haven't actually tested the
case where an error is raised in case a target name is missing. We've been informed of
that, nothing more, nothing less. We simply know we're missing a test here. Isn't this
cool? I think so!</p>
<p>Just remember that code coverage will only <a
href="http://codebetter.com/karlseguin/2008/12/09/code-coverage-use-it-wisely/">bring
you numbers</a> and raw information, not actual proofs that the whole of your <em>code
logic</em> has been actually covered. If you ask me, the best inputs you can get about
your code logic and implementation ever are the ones issued out of <a
href="http://www.extremeprogramming.org/rules/pair.html">pair programming</a>
sessions and <a href="http://alexgaynor.net/2013/sep/26/effective-code-review/">code
reviews</a> — but that's another story.</p>
<p><strong>So is code coverage silver bullet? No. Is it useful? Definitely. Happy
testing!</strong>
</p>
</section>
</div>

331
app/src/main/assets/readability/test/test-pages/001/source.html
View File

@ -1,86 +1,97 @@
<!DOCTYPE html>
<html class="no-js" lang="en">
<head>
<meta charset="utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>Get your Frontend JavaScript Code Covered | Code | Nicolas Perriault</title>
<meta
name="description" content="Nicolas Perriault's homepage."/>
<meta name="viewport" content="width=device-width"/>
<link href="//fonts.googleapis.com/css?family=Asap:400,400italic,700,700italic&amp;subset=latin,latin-ext"
rel="stylesheet" type="text/css"/>
<link rel="stylesheet" type="text/css" href="/static/packed.css?1412806084"/>
<link rel="alternate" type="application/rss+xml" href="/code/feed/" title="Code (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/photography/feed/"
title="Photography (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/talks/feed/" title="Talks (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/carnet/feed/"
title="Carnet (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/feed/" title="Everything (RSS)"/>
<!--[if lt IE 9]>
<script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
</head>
<head>
<meta charset="utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>Get your Frontend JavaScript Code Covered | Code | Nicolas Perriault</title>
<meta
name="description" content="Nicolas Perriault's homepage."/>
<meta name="viewport" content="width=device-width"/>
<link href="//fonts.googleapis.com/css?family=Asap:400,400italic,700,700italic&amp;subset=latin,latin-ext"
rel="stylesheet" type="text/css"/>
<link rel="stylesheet" type="text/css" href="/static/packed.css?1412806084"/>
<link rel="alternate" type="application/rss+xml" href="/code/feed/" title="Code (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/photography/feed/"
title="Photography (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/talks/feed/" title="Talks (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/carnet/feed/"
title="Carnet (RSS)"/>
<link rel="alternate" type="application/rss+xml" href="/feed/" title="Everything (RSS)"/>
<!--[if lt IE 9]>
<script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
</head>
<body class="code " onload="prettyPrint()">
<!--[if lt IE 7]>
<p class="chromeframe">Your browser is <em>ancient!</em> Please <a href="http://www.quirksmode.org/upgrade.html">upgrade</a>.</p>
<![endif]-->
<div class="container">
<header class="main-title">
<h1><a href="/">Hi, I'm <strong>Nicolas.</strong></a></h1>
<small>I code stuff. I take photos. I write rants.</small>
<body class="code " onload="prettyPrint()">
<!--[if lt IE 7]>
<p class="chromeframe">Your browser is <em>ancient!</em> Please <a
href="http://www.quirksmode.org/upgrade.html">upgrade</a>.</p>
<![endif]-->
<div class="container">
<header class="main-title">
<h1><a href="/">Hi, I'm <strong>Nicolas.</strong></a></h1>
<small>I code stuff. I take photos. I write rants.</small>
</header>
<main class="contents" role="main">
<article lang="en" class="code" itemscope="" itemtype="http://schema.org/BlogPosting">
<link itemprop="url" href="/code/2013/get-your-frontend-javascript-code-covered/"/>
<header>
<h2><a itemprop="name" href="/code/2013/get-your-frontend-javascript-code-covered/">Get
your Frontend JavaScript Code Covered</a></h2>
</header>
<main class="contents" role="main">
<article lang="en" class="code" itemscope="" itemtype="http://schema.org/BlogPosting">
<link itemprop="url" href="/code/2013/get-your-frontend-javascript-code-covered/"/>
<header>
<h2><a itemprop="name" href="/code/2013/get-your-frontend-javascript-code-covered/">Get your Frontend JavaScript Code Covered</a></h2>
</header>
<section>
<p><strong>So finally you're <a href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">testing your frontend JavaScript code</a>? Great! The more you
write tests, the more confident you are with your code… but how much precisely?
That's where <a href="http://en.wikipedia.org/wiki/Code_coverage">code coverage</a> might
help.</strong>
</p>
<p>The idea behind code coverage is to record which parts of your code (functions,
statements, conditionals and so on) have been executed by your test suite,
to compute metrics out of these data and usually to provide tools for navigating
and inspecting them.</p>
<p>Not a lot of frontend developers I know actually test their frontend code,
and I can barely imagine how many of them have ever setup code coverage…
Mostly because there are not many frontend-oriented tools in this area
I guess.</p>
<p>Actually I've only found one which provides an adapter for <a href="http://visionmedia.github.io/mocha/">Mocha</a> and
actually works…</p>
<blockquote class="twitter-tweet tw-align-center">
<p>Drinking game for web devs:
<br />(1) Think of a noun
<br />(2) Google "&lt;noun&gt;.js"
<br />(3) If a library with that name exists - drink</p>— Shay Friedman (@ironshay)
<a
href="https://twitter.com/ironshay/statuses/370525864523743232">August 22, 2013</a>
</blockquote>
<p><strong><a href="http://blanketjs.org/">Blanket.js</a></strong> is an <em>easy to install, easy to configure,
and easy to use JavaScript code coverage library that works both in-browser and
with nodejs.</em>
</p>
<p>Its use is dead easy, adding Blanket support to your Mocha test suite
is just matter of adding this simple line to your HTML test file:</p>
<pre><code>&lt;script src="vendor/blanket.js"
<section>
<p><strong>So finally you're <a
href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">testing
your frontend JavaScript code</a>? Great! The more you
write tests, the more confident you are with your code… but how much precisely?
That's where <a href="http://en.wikipedia.org/wiki/Code_coverage">code
coverage</a> might
help.</strong>
</p>
<p>The idea behind code coverage is to record which parts of your code (functions,
statements, conditionals and so on) have been executed by your test suite,
to compute metrics out of these data and usually to provide tools for navigating
and inspecting them.</p>
<p>Not a lot of frontend developers I know actually test their frontend code,
and I can barely imagine how many of them have ever setup code coverage…
Mostly because there are not many frontend-oriented tools in this area
I guess.</p>
<p>Actually I've only found one which provides an adapter for <a
href="http://visionmedia.github.io/mocha/">Mocha</a> and
actually works…</p>
<blockquote class="twitter-tweet tw-align-center">
<p>Drinking game for web devs:
<br/>(1) Think of a noun
<br/>(2) Google "&lt;noun&gt;.js"
<br/>(3) If a library with that name exists - drink</p>— Shay Friedman
(@ironshay)
<a
href="https://twitter.com/ironshay/statuses/370525864523743232">August
22, 2013</a>
</blockquote>
<p><strong><a href="http://blanketjs.org/">Blanket.js</a></strong> is an <em>easy to
install, easy to configure,
and easy to use JavaScript code coverage library that works both in-browser and
with nodejs.</em>
</p>
<p>Its use is dead easy, adding Blanket support to your Mocha test suite
is just matter of adding this simple line to your HTML test file:</p>
<pre><code>&lt;script src="vendor/blanket.js"
data-cover-adapter="vendor/mocha-blanket.js"&gt;&lt;/script&gt;
</code></pre>
<p>Source files: <a href="https://raw.github.com/alex-seville/blanket/master/dist/qunit/blanket.min.js">blanket.js</a>,
<a
<p>Source files: <a
href="https://raw.github.com/alex-seville/blanket/master/dist/qunit/blanket.min.js">blanket.js</a>,
<a
href="https://raw.github.com/alex-seville/blanket/master/src/adapters/mocha-blanket.js">mocha-blanket.js</a>
</p>
<p>As an example, let's reuse the silly <code>Cow</code> example we used
<a
href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">in a previous episode</a>:</p>
<pre><code>// cow.js
</p>
<p>As an example, let's reuse the silly <code>Cow</code> example we used
<a
href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">in
a previous episode</a>:</p>
<pre><code>// cow.js
(function(exports) {
"use strict";
@ -99,8 +110,9 @@ with nodejs.</em>
})(this);
</code></pre>
<p>And its test suite, powered by Mocha and <a href="http://chaijs.com/">Chai</a>:</p>
<pre><code>var expect = chai.expect;
<p>And its test suite, powered by Mocha and <a href="http://chaijs.com/">Chai</a>:
</p>
<pre><code>var expect = chai.expect;
describe("Cow", function() {
describe("constructor", function() {
@ -124,9 +136,9 @@ describe("Cow", function() {
});
</code></pre>
<p>Let's create the HTML test file for it, featuring Blanket and its adapter
for Mocha:</p>
<pre><code>&lt;!DOCTYPE html&gt;
<p>Let's create the HTML test file for it, featuring Blanket and its adapter
for Mocha:</p>
<pre><code>&lt;!DOCTYPE html&gt;
&lt;html&gt;
&lt;head&gt;
&lt;meta charset="utf-8"&gt;
@ -149,85 +161,104 @@ describe("Cow", function() {
&lt;/html&gt;
</code></pre>
<p><strong>Notes</strong>:</p>
<ul>
<li>Notice the <code>data-cover</code> attribute we added to the script tag
loading the source of our library;</li>
<li>The HTML test file <em>must</em> be served over HTTP for the adapter to
be loaded.</li>
</ul>
<p>Running the tests now gives us something like this:</p>
<p>
<img alt="screenshot" src="/static/code/2013/blanket-coverage.png"/>
</p>
<p>As you can see, the report at the bottom highlights that we haven't actually
tested the case where an error is raised in case a target name is missing.
We've been informed of that, nothing more, nothing less. We simply know
we're missing a test here. Isn't this cool? I think so!</p>
<p>Just remember that code coverage will only <a href="http://codebetter.com/karlseguin/2008/12/09/code-coverage-use-it-wisely/">bring you numbers</a> and
raw information, not actual proofs that the whole of your <em>code logic</em> has
been actually covered. If you ask me, the best inputs you can get about
your code logic and implementation ever are the ones issued out of <a href="http://www.extremeprogramming.org/rules/pair.html">pair programming</a>
sessions
and <a href="http://alexgaynor.net/2013/sep/26/effective-code-review/">code reviews</a> —
but that's another story.</p>
<p><strong>So is code coverage silver bullet? No. Is it useful? Definitely. Happy testing!</strong>
</p>
</section>
<aside>
<p> <span class="article-author" itemprop="author" itemscope="" itemtype="http://schema.org/Person">
<span itemprop="name">Nicolas Perriault</span> —</span>
<time
datetime="2013-09-29" itemprop="datePublished">2013-09-29</time>— in <a href="/code/" itemprop="genre">Code</a>
— <a href="/code/2013/get-your-frontend-javascript-code-covered/">Permalink</a>
—
<a
rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/">License</a>— <a href="http://flattr.com/submit/auto?url=https://nicolas.perriault.net/code/2013/get-your-frontend-javascript-code-covered/&amp;title=Get your Frontend JavaScript Code Covered&amp;user_id=n1k0&amp;category=software&amp;language=en">flattr this</a>
</p>
</aside>
<hr/>
<nav> <a class="prev" href="/code/2013/functional-javascript-for-crawling-the-web/">Functional JavaScript for crawling the Web</a>
|
<a
class="next" href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">Testing your frontend JavaScript code using mocha, chai, and sinon</a>
</nav>
</article>
</main>
<nav class="sidebar">
<p><strong>Notes</strong>:</p>
<ul>
<li class="home"><a href="/" hreflang="en">Home</a>
<li>Notice the <code>data-cover</code> attribute we added to the script tag
loading the source of our library;
</li>
<li class="code"><a href="/code/" hreflang="en">Code</a>
</li>
<li class="photography"><a href="/photography/" hreflang="en">Photography</a>
</li>
<li class="talks"><a href="/talks/" hreflang="en">Talks</a>
</li>
<li class="carnet"><a href="/carnet/" hreflang="fr">Carnet <span>fr</span></a>
</li>
<li class="contact"><a href="/contact/" hreflang="en">Contact</a>
<li>The HTML test file <em>must</em> be served over HTTP for the adapter to
be loaded.
</li>
</ul>
</nav>
<footer class="site-footer">
<p>© 2012 Nicolas Perriault — <a href="https://twitter.com/n1k0">Tweet at me</a>
—
<p>Running the tests now gives us something like this:</p>
<p>
<img alt="screenshot" src="/static/code/2013/blanket-coverage.png"/>
</p>
<p>As you can see, the report at the bottom highlights that we haven't actually
tested the case where an error is raised in case a target name is missing.
We've been informed of that, nothing more, nothing less. We simply know
we're missing a test here. Isn't this cool? I think so!</p>
<p>Just remember that code coverage will only <a
href="http://codebetter.com/karlseguin/2008/12/09/code-coverage-use-it-wisely/">bring
you numbers</a> and
raw information, not actual proofs that the whole of your <em>code logic</em>
has
been actually covered. If you ask me, the best inputs you can get about
your code logic and implementation ever are the ones issued out of <a
href="http://www.extremeprogramming.org/rules/pair.html">pair
programming</a>
sessions
and <a href="http://alexgaynor.net/2013/sep/26/effective-code-review/">code
reviews</a> —
but that's another story.</p>
<p><strong>So is code coverage silver bullet? No. Is it useful? Definitely. Happy
testing!</strong>
</p>
</section>
<aside>
<p> <span class="article-author" itemprop="author" itemscope=""
itemtype="http://schema.org/Person">
<span itemprop="name">Nicolas Perriault</span> —</span>
<time
datetime="2013-09-29" itemprop="datePublished">2013-09-29
</time>
— in <a href="/code/" itemprop="genre">Code</a>
— <a href="/code/2013/get-your-frontend-javascript-code-covered/">Permalink</a>
—
<a
href="https://github.com/n1k0">Get my code</a>— <a href="http://500px.com/n1k0">Enjoy my pics</a>
— <a href="/contact/">Contact me</a>
rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/">License</a>—
<a href="http://flattr.com/submit/auto?url=https://nicolas.perriault.net/code/2013/get-your-frontend-javascript-code-covered/&amp;title=Get your Frontend JavaScript Code Covered&amp;user_id=n1k0&amp;category=software&amp;language=en">flattr
this</a>
</p>
</footer>
</div>
<!-- /container -->
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
<script>
</aside>
<hr/>
<nav><a class="prev" href="/code/2013/functional-javascript-for-crawling-the-web/">Functional
JavaScript for crawling the Web</a>
|
<a
class="next"
href="/code/2013/testing-frontend-javascript-code-using-mocha-chai-and-sinon/">Testing
your frontend JavaScript code using mocha, chai, and sinon</a>
</nav>
</article>
</main>
<nav class="sidebar">
<ul>
<li class="home"><a href="/" hreflang="en">Home</a>
</li>
<li class="code"><a href="/code/" hreflang="en">Code</a>
</li>
<li class="photography"><a href="/photography/" hreflang="en">Photography</a>
</li>
<li class="talks"><a href="/talks/" hreflang="en">Talks</a>
</li>
<li class="carnet"><a href="/carnet/" hreflang="fr">Carnet <span>fr</span></a>
</li>
<li class="contact"><a href="/contact/" hreflang="en">Contact</a>
</li>
</ul>
</nav>
<footer class="site-footer">
<p>© 2012 Nicolas Perriault — <a href="https://twitter.com/n1k0">Tweet at me</a>
—
<a
href="https://github.com/n1k0">Get my code</a>— <a href="http://500px.com/n1k0">Enjoy
my pics</a>
— <a href="/contact/">Contact me</a>
</p>
</footer>
</div>
<!-- /container -->
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js"></script>
<script>
window.jQuery || document.write('&lt;script src="js/libs/jquery-1.7.1.min.js">&lt;\/script>')
</script>
<script type="text/javascript" src="/static/js/libs/prettify/prettify.js"></script>
<script type="text/javascript" src="/static/js/app.js"></script>
<script src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
</body>
</script>
<script type="text/javascript" src="/static/js/libs/prettify/prettify.js"></script>
<script type="text/javascript" src="/static/js/app.js"></script>
<script src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
</body>
</html>

271
app/src/main/assets/readability/test/test-pages/002/expected.html
View File

@ -1,18 +1,37 @@
<div id="readability-page-1" class="page">
<div id="content-main">
<article role="article">
<p>For more than a decade the Web has used XMLHttpRequest (XHR) to achieve asynchronous requests in JavaScript. While very useful, XHR is not a very nice API. It suffers from lack of separation of concerns. The input, output and state are all managed by interacting with one object, and state is tracked using events. Also, the event-based model doesn’t play well with JavaScript’s recent focus on Promise- and generator-based asynchronous programming.</p>
<p>The <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch API</a> intends to fix most of these problems. It does this by introducing the same primitives to JS that are used in the HTTP protocol. In addition, it introduces a utility function <code>fetch()</code> that succinctly captures the intention of retrieving a resource from the network.</p>
<p>The <a href="https://fetch.spec.whatwg.org/">Fetch specification</a>, which defines the API, nails down the semantics of a user agent fetching a resource. This, combined with ServiceWorkers, is an attempt to:</p>
<p>For more than a decade the Web has used XMLHttpRequest (XHR) to achieve asynchronous
requests in JavaScript. While very useful, XHR is not a very nice API. It suffers
from lack of separation of concerns. The input, output and state are all managed by
interacting with one object, and state is tracked using events. Also, the
event-based model doesn’t play well with JavaScript’s recent focus on Promise- and
generator-based asynchronous programming.</p>
<p>The <a href="https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API">Fetch
API</a> intends to fix most of these problems. It does this by introducing the same
primitives to JS that are used in the HTTP protocol. In addition, it introduces a
utility function <code>fetch()</code> that succinctly captures the intention of
retrieving a resource from the network.</p>
<p>The <a href="https://fetch.spec.whatwg.org/">Fetch specification</a>, which defines
the API, nails down the semantics of a user agent fetching a resource. This,
combined with ServiceWorkers, is an attempt to:</p>
<ol>
<li>Improve the offline experience.</li>
<li>Expose the building blocks of the Web to the platform as part of the <a href="https://extensiblewebmanifesto.org/">extensible web movement</a>.</li>
<li>Expose the building blocks of the Web to the platform as part of the <a
href="https://extensiblewebmanifesto.org/">extensible web movement</a>.
</li>
</ol>
<p>As of this writing, the Fetch API is available in Firefox 39 (currently Nightly) and Chrome 42 (currently dev). Github has a <a href="https://github.com/github/fetch">Fetch polyfill</a>.</p>
<p>As of this writing, the Fetch API is available in Firefox 39 (currently Nightly) and
Chrome 42 (currently dev). Github has a <a href="https://github.com/github/fetch">Fetch
polyfill</a>.</p>
<h2>Feature detection</h2>
<p>Fetch API support can be detected by checking for <code>Headers</code>,<code>Request</code>, <code>Response</code> or <code>fetch</code> on the <code>window</code> or <code>worker</code> scope.</p>
<p>Fetch API support can be detected by checking for
<code>Headers</code>,<code>Request</code>, <code>Response</code> or
<code>fetch</code> on the <code>window</code> or <code>worker</code> scope.</p>
<h2>Simple fetching</h2>
<p>The most useful, high-level part of the Fetch API is the <code>fetch()</code> function. In its simplest form it takes a URL and returns a promise that resolves to the response. The response is captured as a <code>Response</code> object.</p>
<p>The most useful, high-level part of the Fetch API is the <code>fetch()</code>
function. In its simplest form it takes a URL and returns a promise that resolves to
the response. The response is captured as a <code>Response</code> object.</p>
<div>
<pre>fetch<span>(</span><span>"/data.json"</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>res<span>)</span> <span>{</span>
<span>// res instanceof Response == true.</span>
@ -45,14 +64,20 @@
alert<span>(</span><span>"Error submitting form!"</span><span>)</span><span>;</span>
<span>}</span><span>)</span><span>;</span></pre>
</div>
<p>The <code>fetch()</code> function’s arguments are the same as those passed to the <br />
<code>Request()</code> constructor, so you may directly pass arbitrarily complex requests to <code>fetch()</code> as discussed below.
<p>The <code>fetch()</code> function’s arguments are the same as those passed to the
<br/>
<code>Request()</code> constructor, so you may directly pass arbitrarily complex
requests to <code>fetch()</code> as discussed below.
</p>
<h2>Headers</h2>
<p>Fetch introduces 3 interfaces. These are <code>Headers</code>, <code>Request</code> and <br />
<code>Response</code>. They map directly to the underlying HTTP concepts, but have <br />certain visibility filters in place for privacy and security reasons, such as <br />supporting CORS rules and ensuring cookies aren’t readable by third parties.
<p>Fetch introduces 3 interfaces. These are <code>Headers</code>, <code>Request</code>
and <br/>
<code>Response</code>. They map directly to the underlying HTTP concepts, but have
<br/>certain visibility filters in place for privacy and security reasons, such as
<br/>supporting CORS rules and ensuring cookies aren’t readable by third parties.
</p>
<p>The <a href="https://fetch.spec.whatwg.org/#headers-class">Headers interface</a> is a simple multi-map of names to values:</p>
<p>The <a href="https://fetch.spec.whatwg.org/#headers-class">Headers interface</a> is a
simple multi-map of names to values:</p>
<div>
<pre><span>var</span> content <span>=</span> <span>"Hello World"</span><span>;</span>
<span>var</span> reqHeaders <span>=</span> <span>new</span> Headers<span>(</span><span>)</span><span>;</span>
@ -60,7 +85,8 @@ reqHeaders.<span>append</span><span>(</span><span>"Content-Type"</span><span>,</
reqHeaders.<span>append</span><span>(</span><span>"Content-Length"</span><span>,</span> content.<span>length</span>.<span>toString</span><span>(</span><span>)</span><span>)</span><span>;</span>
reqHeaders.<span>append</span><span>(</span><span>"X-Custom-Header"</span><span>,</span> <span>"ProcessThisImmediately"</span><span>)</span><span>;</span></pre>
</div>
<p>The same can be achieved by passing an array of arrays or a JS object literal <br />to the constructor:</p>
<p>The same can be achieved by passing an array of arrays or a JS object literal <br/>to
the constructor:</p>
<div>
<pre>reqHeaders <span>=</span> <span>new</span> Headers<span>(</span><span>{</span>
<span>"Content-Type"</span><span>:</span> <span>"text/plain"</span><span>,</span>
@ -81,17 +107,34 @@ console.<span>log</span><span>(</span>reqHeaders.<span>getAll</span><span>(</spa
reqHeaders.<span>delete</span><span>(</span><span>"X-Custom-Header"</span><span>)</span><span>;</span>
console.<span>log</span><span>(</span>reqHeaders.<span>getAll</span><span>(</span><span>"X-Custom-Header"</span><span>)</span><span>)</span><span>;</span> <span>// []</span></pre>
</div>
<p>Some of these operations are only useful in ServiceWorkers, but they provide <br />a much nicer API to Headers.</p>
<p>Since Headers can be sent in requests, or received in responses, and have various limitations about what information can and should be mutable, <code>Headers</code> objects have a <strong>guard</strong> property. This is not exposed to the Web, but it affects which mutation operations are allowed on the Headers object. <br />Possible values are:</p>
<p>Some of these operations are only useful in ServiceWorkers, but they provide <br/>a
much nicer API to Headers.</p>
<p>Since Headers can be sent in requests, or received in responses, and have various
limitations about what information can and should be mutable, <code>Headers</code>
objects have a <strong>guard</strong> property. This is not exposed to the Web, but
it affects which mutation operations are allowed on the Headers object. <br/>Possible
values are:</p>
<ul>
<li>“none”: default.</li>
<li>“request”: guard for a Headers object obtained from a Request (<code>Request.headers</code>).</li>
<li>“request-no-cors”: guard for a Headers object obtained from a Request created <br />with mode “no-cors”.</li>
<li>“response”: naturally, for Headers obtained from Response (<code>Response.headers</code>).</li>
<li>“immutable”: Mostly used for ServiceWorkers, renders a Headers object <br />read-only.</li>
<li>“request”: guard for a Headers object obtained from a Request (<code>Request.headers</code>).
</li>
<li>“request-no-cors”: guard for a Headers object obtained from a Request created
<br/>with mode “no-cors”.
</li>
<li>“response”: naturally, for Headers obtained from Response (<code>Response.headers</code>).
</li>
<li>“immutable”: Mostly used for ServiceWorkers, renders a Headers object <br/>read-only.
</li>
</ul>
<p>The details of how each guard affects the behaviors of the Headers object are <br />in the <a href="https://fetch.spec.whatwg.org/">specification</a>. For example, you may not append or set a “request” guarded Headers’ “Content-Length” header. Similarly, inserting “Set-Cookie” into a Response header is not allowed so that ServiceWorkers may not set cookies via synthesized Responses.</p>
<p>All of the Headers methods throw TypeError if <code>name</code> is not a <a href="https://fetch.spec.whatwg.org/#concept-header-name">valid HTTP Header name</a>. The mutation operations will throw TypeError if there is an immutable guard. Otherwise they fail silently. For example:</p>
<p>The details of how each guard affects the behaviors of the Headers object are <br/>in
the <a href="https://fetch.spec.whatwg.org/">specification</a>. For example, you may
not append or set a “request” guarded Headers’ “Content-Length” header. Similarly,
inserting “Set-Cookie” into a Response header is not allowed so that ServiceWorkers
may not set cookies via synthesized Responses.</p>
<p>All of the Headers methods throw TypeError if <code>name</code> is not a <a
href="https://fetch.spec.whatwg.org/#concept-header-name">valid HTTP Header
name</a>. The mutation operations will throw TypeError if there is an immutable
guard. Otherwise they fail silently. For example:</p>
<div>
<pre><span>var</span> res <span>=</span> Response.<span>error</span><span>(</span><span>)</span><span>;</span>
<span>try</span> <span>{</span>
@ -101,21 +144,27 @@ console.<span>log</span><span>(</span>reqHeaders.<span>getAll</span><span>(</spa
<span>}</span></pre>
</div>
<h2>Request</h2>
<p>The Request interface defines a request to fetch a resource over HTTP. URL, method and headers are expected, but the Request also allows specifying a body, a request mode, credentials and cache hints.</p>
<p>The Request interface defines a request to fetch a resource over HTTP. URL, method
and headers are expected, but the Request also allows specifying a body, a request
mode, credentials and cache hints.</p>
<p>The simplest Request is of course, just a URL, as you may do to GET a resource.</p>
<div>
<pre><span>var</span> req <span>=</span> <span>new</span> Request<span>(</span><span>"/index.html"</span><span>)</span><span>;</span>
console.<span>log</span><span>(</span>req.<span>method</span><span>)</span><span>;</span> <span>// "GET"</span>
console.<span>log</span><span>(</span>req.<span>url</span><span>)</span><span>;</span> <span>// "http://example.com/index.html"</span></pre>
</div>
<p>You may also pass a Request to the <code>Request()</code> constructor to create a copy. <br />(This is not the same as calling the <code>clone()</code> method, which is covered in <br />the “Reading bodies” section.).</p>
<p>You may also pass a Request to the <code>Request()</code> constructor to create a
copy. <br/>(This is not the same as calling the <code>clone()</code> method, which
is covered in <br/>the “Reading bodies” section.).</p>
<div>
<pre><span>var</span> copy <span>=</span> <span>new</span> Request<span>(</span>req<span>)</span><span>;</span>
console.<span>log</span><span>(</span>copy.<span>method</span><span>)</span><span>;</span> <span>// "GET"</span>
console.<span>log</span><span>(</span>copy.<span>url</span><span>)</span><span>;</span> <span>// "http://example.com/index.html"</span></pre>
</div>
<p>Again, this form is probably only useful in ServiceWorkers.</p>
<p>The non-URL attributes of the <code>Request</code> can only be set by passing initial <br />values as a second argument to the constructor. This argument is a dictionary.</p>
<p>The non-URL attributes of the <code>Request</code> can only be set by passing initial
<br/>values as a second argument to the constructor. This argument is a dictionary.
</p>
<div>
<pre><span>var</span> uploadReq <span>=</span> <span>new</span> Request<span>(</span><span>"/uploadImage"</span><span>,</span> <span>{</span>
method<span>:</span> <span>"POST"</span><span>,</span>
@ -125,8 +174,13 @@ console.<span>log</span><span>(</span>copy.<span>url</span><span>)</span><span>;
body<span>:</span> <span>"image data"</span>
<span>}</span><span>)</span><span>;</span></pre>
</div>
<p>The Request’s mode is used to determine if cross-origin requests lead to valid responses, and which properties on the response are readable. Legal mode values are <code>"same-origin"</code>, <code>"no-cors"</code> (default) and <code>"cors"</code>.</p>
<p>The <code>"same-origin"</code> mode is simple, if a request is made to another origin with this mode set, the result is simply an error. You could use this to ensure that <br />a request is always being made to your origin.</p>
<p>The Request’s mode is used to determine if cross-origin requests lead to valid
responses, and which properties on the response are readable. Legal mode values are
<code>"same-origin"</code>, <code>"no-cors"</code> (default) and <code>"cors"</code>.
</p>
<p>The <code>"same-origin"</code> mode is simple, if a request is made to another origin
with this mode set, the result is simply an error. You could use this to ensure that
<br/>a request is always being made to your origin.</p>
<div>
<pre><span>var</span> arbitraryUrl <span>=</span> document.<span>getElementById</span><span>(</span><span>"url-input"</span><span>)</span>.<span>value</span><span>;</span>
fetch<span>(</span>arbitraryUrl<span>,</span> <span>{</span> mode<span>:</span> <span>"same-origin"</span> <span>}</span><span>)</span>.<span>then</span><span>(</span><span>function</span><span>(</span>res<span>)</span> <span>{</span>
@ -135,8 +189,24 @@ fetch<span>(</span>arbitraryUrl<span>,</span> <span>{</span> mode<span>:</span>
console.<span>log</span><span>(</span><span>"Please enter a same-origin URL!"</span><span>)</span><span>;</span>
<span>}</span><span>)</span><span>;</span></pre>
</div>
<p>The <code>"no-cors"</code> mode captures what the web platform does by default for scripts you import from CDNs, images hosted on other domains, and so on. First, it prevents the method from being anything other than “HEAD”, “GET” or “POST”. Second, if any ServiceWorkers intercept these requests, they may not add or override any headers except for <a href="https://fetch.spec.whatwg.org/#simple-header">these</a>. Third, JavaScript may not access any properties of the resulting Response. This ensures that ServiceWorkers do not affect the semantics of the Web and prevents security and privacy issues that could arise from leaking data across domains.</p>
<p><code>"cors"</code> mode is what you’ll usually use to make known cross-origin requests to access various APIs offered by other vendors. These are expected to adhere to <br />the <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS">CORS protocol</a>. Only a <a href="https://fetch.spec.whatwg.org/#concept-filtered-response-cors">limited set</a> of headers is exposed in the Response, but the body is readable. For example, you could get a list of Flickr’s <a href="https://www.flickr.com/services/api/flickr.interestingness.getList.html">most interesting</a> photos today like this:</p>
<p>The <code>"no-cors"</code> mode captures what the web platform does by default for
scripts you import from CDNs, images hosted on other domains, and so on. First, it
prevents the method from being anything other than “HEAD”, “GET” or “POST”. Second,
if any ServiceWorkers intercept these requests, they may not add or override any
headers except for <a href="https://fetch.spec.whatwg.org/#simple-header">these</a>.
Third, JavaScript may not access any properties of the resulting Response. This
ensures that ServiceWorkers do not affect the semantics of the Web and prevents
security and privacy issues that could arise from leaking data across domains.</p>
<p><code>"cors"</code> mode is what you’ll usually use to make known cross-origin
requests to access various APIs offered by other vendors. These are expected to
adhere to <br/>the <a
href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS">CORS
protocol</a>. Only a <a
href="https://fetch.spec.whatwg.org/#concept-filtered-response-cors">limited
set</a> of headers is exposed in the Response, but the body is readable. For
example, you could get a list of Flickr’s <a
href="https://www.flickr.com/services/api/flickr.interestingness.getList.html">most
interesting</a> photos today like this:</p>
<div>
<pre><span>var</span> u <span>=</span> <span>new</span> URLSearchParams<span>(</span><span>)</span><span>;</span>
u.<span>append</span><span>(</span><span>'method'</span><span>,</span> <span>'flickr.interestingness.getList'</span><span>)</span><span>;</span>
@ -157,33 +227,65 @@ apiCall.<span>then</span><span>(</span><span>function</span><span>(</span>respon
<span>}</span><span>)</span><span>;</span>
<span>}</span><span>)</span><span>;</span></pre>
</div>
<p>You may not read out the “Date” header since Flickr does not allow it via <br />
<p>You may not read out the “Date” header since Flickr does not allow it via <br/>
<code>Access-Control-Expose-Headers</code>.
</p>
<div>
<pre>response.<span>headers</span>.<span>get</span><span>(</span><span>"Date"</span><span>)</span><span>;</span> <span>// null</span></pre>
</div>
<p>The <code>credentials</code> enumeration determines if cookies for the other domain are <br />sent to cross-origin requests. This is similar to XHR’s <code>withCredentials</code>
<br />flag, but tri-valued as <code>"omit"</code> (default), <code>"same-origin"</code> and <code>"include"</code>.
<p>The <code>credentials</code> enumeration determines if cookies for the other domain
are <br/>sent to cross-origin requests. This is similar to XHR’s <code>withCredentials</code>
<br/>flag, but tri-valued as <code>"omit"</code> (default),
<code>"same-origin"</code> and <code>"include"</code>.
</p>
<p>The Request object will also give the ability to offer caching hints to the user-agent. This is currently undergoing some <a href="https://github.com/slightlyoff/ServiceWorker/issues/585">security review</a>. Firefox exposes the attribute, but it has no effect.</p>
<p>Requests have two read-only attributes that are relevant to ServiceWorkers <br />intercepting them. There is the string <code>referrer</code>, which is set by the UA to be <br />the referrer of the Request. This may be an empty string. The other is <br />
<code>context</code> which is a rather <a href="https://fetch.spec.whatwg.org/#requestcredentials">large enumeration</a> defining what sort of resource is being fetched. This could be “image” if the request is from an &lt;img&gt;tag in the controlled document, “worker” if it is an attempt to load a worker script, and so on. When used with the <code>fetch()</code> function, it is “fetch”.
<p>The Request object will also give the ability to offer caching hints to the
user-agent. This is currently undergoing some <a
href="https://github.com/slightlyoff/ServiceWorker/issues/585">security
review</a>. Firefox exposes the attribute, but it has no effect.</p>
<p>Requests have two read-only attributes that are relevant to ServiceWorkers <br/>intercepting
them. There is the string <code>referrer</code>, which is set by the UA to be <br/>the
referrer of the Request. This may be an empty string. The other is <br/>
<code>context</code> which is a rather <a
href="https://fetch.spec.whatwg.org/#requestcredentials">large
enumeration</a> defining what sort of resource is being fetched. This could be
“image” if the request is from an &lt;img&gt;tag in the controlled document,
“worker” if it is an attempt to load a worker script, and so on. When used with the
<code>fetch()</code> function, it is “fetch”.
</p>
<h2>Response</h2>
<p><code>Response</code> instances are returned by calls to <code>fetch()</code>. They can also be created by JS, but this is only useful in ServiceWorkers.</p>
<p>We have already seen some attributes of Response when we looked at <code>fetch()</code>. The most obvious candidates are <code>status</code>, an integer (default value 200) and <code>statusText</code> (default value “OK”), which correspond to the HTTP status code and reason. The <code>ok</code> attribute is just a shorthand for checking that <code>status</code> is in the range 200-299 inclusive.</p>
<p><code>headers</code> is the Response’s Headers object, with guard “response”. The <code>url</code> attribute reflects the URL of the corresponding request.</p>
<p>Response also has a <code>type</code>, which is “basic”, “cors”, “default”, “error” or <br />“opaque”.</p>
<p><code>Response</code> instances are returned by calls to <code>fetch()</code>. They
can also be created by JS, but this is only useful in ServiceWorkers.</p>
<p>We have already seen some attributes of Response when we looked at
<code>fetch()</code>. The most obvious candidates are <code>status</code>, an
integer (default value 200) and <code>statusText</code> (default value “OK”), which
correspond to the HTTP status code and reason. The <code>ok</code> attribute is just
a shorthand for checking that <code>status</code> is in the range 200-299 inclusive.
</p>
<p><code>headers</code> is the Response’s Headers object, with guard “response”. The
<code>url</code> attribute reflects the URL of the corresponding request.</p>
<p>Response also has a <code>type</code>, which is “basic”, “cors”, “default”, “error”
or <br/>“opaque”.</p>
<ul>
<li><code>"basic"</code>: normal, same origin response, with all headers exposed except <br />“Set-Cookie” and “Set-Cookie2″.</li>
<li><code>"cors"</code>: response was received from a valid cross-origin request. <a href="https://fetch.spec.whatwg.org/#concept-filtered-response-cors">Certain headers and the body</a>may be accessed.</li>
<li><code>"error"</code>: network error. No useful information describing the error is available. The Response’s status is 0, headers are empty and immutable. This is the type for a Response obtained from <code>Response.error()</code>.</li>
<li><code>"opaque"</code>: response for “no-cors” request to cross-origin resource. <a href="https://fetch.spec.whatwg.org/#concept-filtered-response-opaque">Severely<br /> restricted</a>
<li><code>"basic"</code>: normal, same origin response, with all headers exposed
except <br/>“Set-Cookie” and “Set-Cookie2″.
</li>
<li><code>"cors"</code>: response was received from a valid cross-origin request. <a
href="https://fetch.spec.whatwg.org/#concept-filtered-response-cors">Certain
headers and the body</a>may be accessed.
</li>
<li><code>"error"</code>: network error. No useful information describing the error
is available. The Response’s status is 0, headers are empty and immutable. This
is the type for a Response obtained from <code>Response.error()</code>.
</li>
<li><code>"opaque"</code>: response for “no-cors” request to cross-origin resource.
<a href="https://fetch.spec.whatwg.org/#concept-filtered-response-opaque">Severely<br/>
restricted</a>
</li>
</ul>
<p>The “error” type results in the <code>fetch()</code> Promise rejecting with TypeError.</p>
<p>There are certain attributes that are useful only in a ServiceWorker scope. The <br />idiomatic way to return a Response to an intercepted request in ServiceWorkers is:</p>
<p>The “error” type results in the <code>fetch()</code> Promise rejecting with
TypeError.</p>
<p>There are certain attributes that are useful only in a ServiceWorker scope. The <br/>idiomatic
way to return a Response to an intercepted request in ServiceWorkers is:</p>
<div>
<pre>addEventListener<span>(</span><span>'fetch'</span><span>,</span> <span>function</span><span>(</span>event<span>)</span> <span>{</span>
event.<span>respondWith</span><span>(</span><span>new</span> Response<span>(</span><span>"Response body"</span><span>,</span> <span>{</span>
@ -191,23 +293,40 @@ apiCall.<span>then</span><span>(</span><span>function</span><span>(</span>respon
<span>}</span><span>)</span><span>;</span>
<span>}</span><span>)</span><span>;</span></pre>
</div>
<p>As you can see, Response has a two argument constructor, where both arguments are optional. The first argument is a body initializer, and the second is a dictionary to set the <code>status</code>, <code>statusText</code> and <code>headers</code>.</p>
<p>The static method <code>Response.error()</code> simply returns an error response. Similarly, <code>Response.redirect(url, status)</code> returns a Response resulting in <br />a redirect to <code>url</code>.</p>
<p>As you can see, Response has a two argument constructor, where both arguments are
optional. The first argument is a body initializer, and the second is a dictionary
to set the <code>status</code>, <code>statusText</code> and <code>headers</code>.
</p>
<p>The static method <code>Response.error()</code> simply returns an error response.
Similarly, <code>Response.redirect(url, status)</code> returns a Response resulting
in <br/>a redirect to <code>url</code>.</p>
<h2>Dealing with bodies</h2>
<p>Both Requests and Responses may contain body data. We’ve been glossing over it because of the various data types body may contain, but we will cover it in detail now.</p>
<p>Both Requests and Responses may contain body data. We’ve been glossing over it
because of the various data types body may contain, but we will cover it in detail
now.</p>
<p>A body is an instance of any of the following types.</p>
<ul>
<li><a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer">ArrayBuffer</a>
<li>
<a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer">ArrayBuffer</a>
</li>
<li><a href="https://developer.mozilla.org/en-US/docs/Web/API/ArrayBufferView">ArrayBufferView</a> (Uint8Array and friends)</li>
<li><a href="https://developer.mozilla.org/en-US/docs/Web/API/Blob">Blob</a>/ <a href="https://developer.mozilla.org/en-US/docs/Web/API/File">File</a>
<li><a href="https://developer.mozilla.org/en-US/docs/Web/API/ArrayBufferView">ArrayBufferView</a>
(Uint8Array and friends)
</li>
<li><a href="https://developer.mozilla.org/en-US/docs/Web/API/Blob">Blob</a>/ <a
href="https://developer.mozilla.org/en-US/docs/Web/API/File">File</a>
</li>
<li>string</li>
<li><a href="https://url.spec.whatwg.org/#interface-urlsearchparams">URLSearchParams</a>
<li>
<a href="https://url.spec.whatwg.org/#interface-urlsearchparams">URLSearchParams</a>
</li>
<li><a href="https://developer.mozilla.org/en-US/docs/Web/API/FormData">FormData</a>
– currently not supported by either Gecko or Blink. Firefox expects to ship this
in version 39 along with the rest of Fetch.
</li>
<li><a href="https://developer.mozilla.org/en-US/docs/Web/API/FormData">FormData</a> – currently not supported by either Gecko or Blink. Firefox expects to ship this in version 39 along with the rest of Fetch.</li>
</ul>
<p>In addition, Request and Response both offer the following methods to extract their body. These all return a Promise that is eventually resolved with the actual content.</p>
<p>In addition, Request and Response both offer the following methods to extract their
body. These all return a Promise that is eventually resolved with the actual
content.</p>
<ul>
<li><code>arrayBuffer()</code>
</li>
@ -220,7 +339,8 @@ apiCall.<span>then</span><span>(</span><span>function</span><span>(</span>respon
<li><code>formData()</code>
</li>
</ul>
<p>This is a significant improvement over XHR in terms of ease of use of non-text data!</p>
<p>This is a significant improvement over XHR in terms of ease of use of non-text
data!</p>
<p>Request bodies can be set by passing <code>body</code> parameters:</p>
<div>
<pre><span>var</span> form <span>=</span> <span>new</span> FormData<span>(</span>document.<span>getElementById</span><span>(</span><span>'login-form'</span><span>)</span><span>)</span><span>;</span>
@ -234,9 +354,15 @@ fetch<span>(</span><span>"/login"</span><span>,</span> <span>{</span>
<pre><span>var</span> res <span>=</span> <span>new</span> Response<span>(</span><span>new</span> File<span>(</span><span>[</span><span>"chunk"</span><span>,</span> <span>"chunk"</span><span>]</span><span>,</span> <span>"archive.zip"</span><span>,</span>
<span>{</span> type<span>:</span> <span>"application/zip"</span> <span>}</span><span>)</span><span>)</span><span>;</span></pre>
</div>
<p>Both Request and Response (and by extension the <code>fetch()</code> function), will try to intelligently <a href="https://fetch.spec.whatwg.org/#concept-bodyinit-extract">determine the content type</a>. Request will also automatically set a “Content-Type” header if none is set in the dictionary.</p>
<p>Both Request and Response (and by extension the <code>fetch()</code> function), will
try to intelligently <a
href="https://fetch.spec.whatwg.org/#concept-bodyinit-extract">determine the
content type</a>. Request will also automatically set a “Content-Type” header if
none is set in the dictionary.</p>
<h3>Streams and cloning</h3>
<p>It is important to realise that Request and Response bodies can only be read once! Both interfaces have a boolean attribute <code>bodyUsed</code> to determine if it is safe to read or not.</p>
<p>It is important to realise that Request and Response bodies can only be read once!
Both interfaces have a boolean attribute <code>bodyUsed</code> to determine if it is
safe to read or not.</p>
<div>
<pre><span>var</span> res <span>=</span> <span>new</span> Response<span>(</span><span>"one time use"</span><span>)</span><span>;</span>
console.<span>log</span><span>(</span>res.<span>bodyUsed</span><span>)</span><span>;</span> <span>// false</span>
@ -249,9 +375,21 @@ res.<span>text</span><span>(</span><span>)</span>.<span>catch</span><span>(</spa
console.<span>log</span><span>(</span><span>"Tried to read already consumed Response"</span><span>)</span><span>;</span>
<span>}</span><span>)</span><span>;</span></pre>
</div>
<p>This decision allows easing the transition to an eventual <a href="https://streams.spec.whatwg.org/">stream-based</a> Fetch API. The intention is to let applications consume data as it arrives, allowing for JavaScript to deal with larger files like videos, and perform things like compression and editing on the fly.</p>
<p>Often, you’ll want access to the body multiple times. For example, you can use the upcoming <a href="http://slightlyoff.github.io/ServiceWorker/spec/service_worker/index.html#cache-objects">Cache API</a> to store Requests and Responses for offline use, and Cache requires bodies to be available for reading.</p>
<p>So how do you read out the body multiple times within such constraints? The API provides a <code>clone()</code> method on the two interfaces. This will return a clone of the object, with a ‘new’ body. <code>clone()</code> MUST be called before the body of the corresponding object has been used. That is, <code>clone()</code> first, read later.</p>
<p>This decision allows easing the transition to an eventual <a
href="https://streams.spec.whatwg.org/">stream-based</a> Fetch API. The
intention is to let applications consume data as it arrives, allowing for JavaScript
to deal with larger files like videos, and perform things like compression and
editing on the fly.</p>
<p>Often, you’ll want access to the body multiple times. For example, you can use the
upcoming <a
href="http://slightlyoff.github.io/ServiceWorker/spec/service_worker/index.html#cache-objects">Cache
API</a> to store Requests and Responses for offline use, and Cache requires
bodies to be available for reading.</p>
<p>So how do you read out the body multiple times within such constraints? The API
provides a <code>clone()</code> method on the two interfaces. This will return a
clone of the object, with a ‘new’ body. <code>clone()</code> MUST be called before
the body of the corresponding object has been used. That is, <code>clone()</code>
first, read later.</p>
<div>
<pre>addEventListener<span>(</span><span>'fetch'</span><span>,</span> <span>function</span><span>(</span>evt<span>)</span> <span>{</span>
<span>var</span> sheep <span>=</span> <span>new</span> Response<span>(</span><span>"Dolly"</span><span>)</span><span>;</span>
@ -269,10 +407,19 @@ res.<span>text</span><span>(</span><span>)</span>.<span>catch</span><span>(</spa
<span>}</span><span>)</span><span>;</span></pre>
</div>
<h2>Future improvements</h2>
<p>Along with the transition to streams, Fetch will eventually have the ability to abort running <code>fetch()</code>es and some way to report the progress of a fetch. These are provided by XHR, but are a little tricky to fit in the Promise-based nature of the Fetch API.</p>
<p>You can contribute to the evolution of this API by participating in discussions on the <a href="https://whatwg.org/mailing-list">WHATWG mailing list</a> and in the issues in the <a href="https://www.w3.org/Bugs/Public/buglist.cgi?product=WHATWG&amp;component=Fetch&amp;resolution=---">Fetch</a> and <a href="https://github.com/slightlyoff/ServiceWorker/issues">ServiceWorker</a>specifications.</p>
<p>Along with the transition to streams, Fetch will eventually have the ability to abort
running <code>fetch()</code>es and some way to report the progress of a fetch. These
are provided by XHR, but are a little tricky to fit in the Promise-based nature of
the Fetch API.</p>
<p>You can contribute to the evolution of this API by participating in discussions on
the <a href="https://whatwg.org/mailing-list">WHATWG mailing list</a> and in the
issues in the <a
href="https://www.w3.org/Bugs/Public/buglist.cgi?product=WHATWG&amp;component=Fetch&amp;resolution=---">Fetch</a>
and <a href="https://github.com/slightlyoff/ServiceWorker/issues">ServiceWorker</a>specifications.
</p>
<p>For a better web!</p>
<p><em>The author would like to thank Andrea Marchesini, Anne van Kesteren and Ben<br /> Kelly for helping with the specification and implementation.</em>
<p><em>The author would like to thank Andrea Marchesini, Anne van Kesteren and Ben<br/>
Kelly for helping with the specification and implementation.</em>
</p>
</article>
</div>

2805
app/src/main/assets/readability/test/test-pages/002/source.html
View File

File diff suppressed because it is too large Load Diff

14
app/src/main/assets/readability/test/test-pages/003-metadata-preferred/expected.html
View File

@ -1,7 +1,17 @@
<div id="readability-page-1" class="page">
<article>
<h2>Test document title</h2>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum. </p>
</article>
</div>

48
app/src/main/assets/readability/test/test-pages/003-metadata-preferred/source.html
View File

@ -1,6 +1,6 @@
<!DOCTYPE html>
<html>
<head>
<head>
<meta charset="utf-8"/>
<title>Title Element</title>
<meta name="title" content="Meta name title"/>
@ -13,33 +13,33 @@
<meta name="author" content="Meta name author"/>
<meta name="DC.creator" content="Dublin Core name author"/>
<meta property="dc:creator" content="Dublin Core property author"/>
<meta name="description" content="Meta name description"/>
<meta name="description" content="Meta name description"/>
<meta name="og:description" content="Open Graph name description"/>
<meta name="twitter:description" content="Twitter name description"/>
<meta name="DC.description" content="Dublin Core name description"/>
<meta property="dc:description" content="Dublin Core property description"/>
<meta property="twitter:description" content="Twitter property description"/>
<meta property="og:description" content="Open Graph property description"/>
</head>
<body>
<article>
<h1>Test document title</h1>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
</article>
</body>
</head>
<body>
<article>
<h1>Test document title</h1>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
</article>
</body>
</html>

14
app/src/main/assets/readability/test/test-pages/004-metadata-space-separated-properties/expected.html
View File

@ -1,7 +1,17 @@
<div id="readability-page-1" class="page">
<article>
<h2>Test document title</h2>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum. </p>
</article>
</div>

46
app/src/main/assets/readability/test/test-pages/004-metadata-space-separated-properties/source.html
View File

@ -1,6 +1,6 @@
<!DOCTYPE html>
<html>
<head>
<head>
<meta charset="utf-8"/>
<title>Title Element</title>
<meta property="x:title dc:title" content="Preferred title"/>
@ -10,26 +10,26 @@
<meta property="og:description x:description twitter:description" content="A description"/>
<meta property="dc:description og:description" content="Preferred description"/>
<meta name="description" content="FAIL"/>
</head>
<body>
<article>
<h1>Test document title</h1>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
</article>
</body>
</head>
<body>
<article>
<h1>Test document title</h1>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
</article>
</body>
</html>

202
app/src/main/assets/readability/test/test-pages/aclu/expected.html
View File

@ -1,61 +1,199 @@
<div id="readability-page-1" class="page">
<div>
<p> I don't use Facebook. I'm not technophobic — I'm a geek. I've been using email since the early 1990s, I have accounts on hundreds of services around the net, and I do software development and internet protocol design both for work and for fun. I believe that a globe-spanning communications network like the internet can be a positive social force, and I publish much of my own work on the open web. </p>
<p> But Facebook and other massive web companies represent a strong push toward unaccountable centralized social control, which I think makes our society more unequal and more unjust. The Cambridge Analytica scandal is one instance of this long-running problem with what I call the "surveillance economy." I don't want to submit to these power structures, and I don’t want my presence on such platforms to serve as bait that lures other people into the digital panopticon. </p>
<p> But while I've never "opted in" to Facebook or any of the other big social networks, Facebook still has a detailed profile that can be used to target me. I've never consented to having Facebook collect my data, which can be used to draw very detailed inferences about my life, my habits, and my relationships. As we aim to take Facebook to task for its breach of user trust, we need to think about what its capabilities imply for society overall. After all, if you do #deleteFacebook, you'll find yourself in my shoes: non-consenting, but still subject to Facebook’s globe-spanning surveillance and targeting network. </p>
<p> There are at least two major categories of information available to Facebook about non-participants like me: information from other Facebook users, and information from sites on the open web. </p>
<p> I don't use Facebook. I'm not technophobic — I'm a geek. I've been using email since the
early 1990s, I have accounts on hundreds of services around the net, and I do software
development and internet protocol design both for work and for fun. I believe that a
globe-spanning communications network like the internet can be a positive social force,
and I publish much of my own work on the open web. </p>
<p> But Facebook and other massive web companies represent a strong push toward
unaccountable centralized social control, which I think makes our society more unequal
and more unjust. The Cambridge Analytica scandal is one instance of this long-running
problem with what I call the "surveillance economy." I don't want to submit to these
power structures, and I don’t want my presence on such platforms to serve as bait that
lures other people into the digital panopticon. </p>
<p> But while I've never "opted in" to Facebook or any of the other big social networks,
Facebook still has a detailed profile that can be used to target me. I've never
consented to having Facebook collect my data, which can be used to draw very detailed
inferences about my life, my habits, and my relationships. As we aim to take Facebook to
task for its breach of user trust, we need to think about what its capabilities imply
for society overall. After all, if you do #deleteFacebook, you'll find yourself in my
shoes: non-consenting, but still subject to Facebook’s globe-spanning surveillance and
targeting network. </p>
<p> There are at least two major categories of information available to Facebook about
non-participants like me: information from other Facebook users, and information from
sites on the open web. </p>
<h3>
<strong>Information from other Facebook users</strong>
</h3>
<p> When you sign up for Facebook, it encourages you to upload your list of contacts so that the site can "find your friends." Facebook uses this contact information to learn about people, even if those people don't agree to participate. It also links people together based on who they know, even if the shared contact hasn't agreed to this use. </p>
<p> For example, I received an email from Facebook that lists the people who have all invited me to join Facebook: my aunt, an old co-worker, a friend from elementary school, etc. This email includes names and email addresses — including my own name — and at least one <a href="https://en.wikipedia.org/wiki/Web_bug">web bug</a> designed to identify me to Facebook’s web servers when I open the email. Facebook records this group of people as my contacts, even though I've never agreed to this kind of data collection. </p>
<p> Similarly, I'm sure that I'm in some photographs that someone has uploaded to Facebook — and I'm probably tagged in some of them. I've never agreed to this, but Facebook could still be keeping track. </p>
<p> So even if you decide you need to join Facebook, remember that you might be giving the company information about someone else who didn't agree to be part of its surveillance platform. </p>
<p> When you sign up for Facebook, it encourages you to upload your list of contacts so that
the site can "find your friends." Facebook uses this contact information to learn about
people, even if those people don't agree to participate. It also links people together
based on who they know, even if the shared contact hasn't agreed to this use. </p>
<p> For example, I received an email from Facebook that lists the people who have all
invited me to join Facebook: my aunt, an old co-worker, a friend from elementary school,
etc. This email includes names and email addresses — including my own name — and at
least one <a href="https://en.wikipedia.org/wiki/Web_bug">web bug</a> designed to
identify me to Facebook’s web servers when I open the email. Facebook records this group
of people as my contacts, even though I've never agreed to this kind of data collection.
</p>
<p> Similarly, I'm sure that I'm in some photographs that someone has uploaded to Facebook —
and I'm probably tagged in some of them. I've never agreed to this, but Facebook could
still be keeping track. </p>
<p> So even if you decide you need to join Facebook, remember that you might be giving the
company information about someone else who didn't agree to be part of its surveillance
platform. </p>
<h3>
<strong>Information from sites on the open Web</strong>
</h3>
<p> Nearly every website that you visit that has a "Like" button is actually encouraging your browser to tell Facebook about your browsing habits. Even if you don't click on the "Like" button, displaying it requires your browser to send a request to Facebook's servers for the "Like" button itself. That request includes <a href="https://en.wikipedia.org/wiki/HTTP_referer">information</a> mentioning the name of the page you are visiting and any Facebook-specific <a href="https://en.wikipedia.org/wiki/HTTP_cookie">cookies</a> your browser might have collected. (See <a href="https://www.facebook.com/help/186325668085084">Facebook's own description of this process</a>.) This is called a "third-party request." </p>
<p> This makes it possible for Facebook to create a detailed picture of your browsing history — even if you've never even visited Facebook directly, let alone signed up for a Facebook account. </p>
<p> Think about most of the web pages you've visited — how many of them <em>don't</em> have a "Like" button? If you administer a website and you include a "Like" button on every page, you're helping Facebook to build profiles of your visitors, even those who have opted out of the social network. Facebook’s <a href="https://developers.facebook.com/docs/plugins/">“Share” buttons</a> on other sites — along with <a href="https://www.facebook.com/business/learn/facebook-ads-pixel">other tools</a> — work a bit differently from the “Like” button, but do effectively the same thing. </p>
<p> The profiles that Facebook builds on non-users don't necessarily include so-called "personally identifiable information" (PII) like names or email addresses. But they do include fairly unique patterns. Using <a href="https://dev.chromium.org/for-testers/providing-network-details">Chromium's NetLog dumping</a>, I performed a simple five-minute browsing test last week that included visits to various sites — but not Facebook. In that test, the PII-free data that was sent to Facebook included information about which news articles I was reading, my dietary preferences, and my hobbies. </p>
<p> Given the precision of this kind of mapping and targeting, "PII" isn’t necessary to reveal my identity. How many vegans examine specifications for computer hardware from the ACLU's offices while reading about Cambridge Analytica? Anyway, if Facebook combined that information with the "web bug" from the email mentioned above — which <em>is</em> clearly linked to my name and e-mail address — no guesswork would be required. </p>
<p> I'd be shocked if Facebook were not connecting those dots given the goals <a href="https://www.facebook.com/about/privacy/cookies">they claim for data collection</a>: </p>
<p> Nearly every website that you visit that has a "Like" button is actually encouraging
your browser to tell Facebook about your browsing habits. Even if you don't click on the
"Like" button, displaying it requires your browser to send a request to Facebook's
servers for the "Like" button itself. That request includes <a
href="https://en.wikipedia.org/wiki/HTTP_referer">information</a> mentioning the
name of the page you are visiting and any Facebook-specific <a
href="https://en.wikipedia.org/wiki/HTTP_cookie">cookies</a> your browser might
have collected. (See <a href="https://www.facebook.com/help/186325668085084">Facebook's
own description of this process</a>.) This is called a "third-party request." </p>
<p> This makes it possible for Facebook to create a detailed picture of your browsing
history — even if you've never even visited Facebook directly, let alone signed up for a
Facebook account. </p>
<p> Think about most of the web pages you've visited — how many of them <em>don't</em> have
a "Like" button? If you administer a website and you include a "Like" button on every
page, you're helping Facebook to build profiles of your visitors, even those who have
opted out of the social network. Facebook’s <a
href="https://developers.facebook.com/docs/plugins/">“Share” buttons</a> on
other sites — along with <a
href="https://www.facebook.com/business/learn/facebook-ads-pixel">other
tools</a> — work a bit differently from the “Like” button, but do effectively the
same thing. </p>
<p> The profiles that Facebook builds on non-users don't necessarily include so-called
"personally identifiable information" (PII) like names or email addresses. But they do
include fairly unique patterns. Using <a
href="https://dev.chromium.org/for-testers/providing-network-details">Chromium's
NetLog dumping</a>, I performed a simple five-minute browsing test last week that
included visits to various sites — but not Facebook. In that test, the PII-free data
that was sent to Facebook included information about which news articles I was reading,
my dietary preferences, and my hobbies. </p>
<p> Given the precision of this kind of mapping and targeting, "PII" isn’t necessary to
reveal my identity. How many vegans examine specifications for computer hardware from
the ACLU's offices while reading about Cambridge Analytica? Anyway, if Facebook combined
that information with the "web bug" from the email mentioned above — which <em>is</em>
clearly linked to my name and e-mail address — no guesswork would be required. </p>
<p> I'd be shocked if Facebook were not connecting those dots given the goals <a
href="https://www.facebook.com/about/privacy/cookies">they claim for data
collection</a>: </p>
<blockquote>
<p> We use the information we have to improve our advertising and measurement systems so we can show you relevant ads on and off our Services and measure the effectiveness and reach of ads and services. </p>
<p> We use the information we have to improve our advertising and measurement systems so
we can show you relevant ads on and off our Services and measure the effectiveness
and reach of ads and services. </p>
</blockquote>
<p> This is, in essence, exactly what Cambridge Analytica did. </p>
<h3>
<strong>Consent</strong>
</h3>
<p> Facebook and other tech companies often deflect accusations against excessive data collection by arguing "consent" — that they harvest and use data with the consent of the users involved. </p>
<p> But even if we accept that clicking through a "Terms of Service" that <a href="https://tosdr.org/">no one reads</a> can actually constitute true consent, even if we ignore the fact that these terms are overwhelmingly one-sided and non-negotiable, and even if we accept that it's meaningful for people to give consent when sharing data about other people who may have also opted in — what is the recourse for someone who has not opted into these systems at all? </p>
<p> Are those of us who have explicitly avoided agreeing to the Facebook terms of service simply fair game for an industry-wide surveillance and targeting network? </p>
<p> Facebook and other tech companies often deflect accusations against excessive data
collection by arguing "consent" — that they harvest and use data with the consent of the
users involved. </p>
<p> But even if we accept that clicking through a "Terms of Service" that <a
href="https://tosdr.org/">no one reads</a> can actually constitute true consent,
even if we ignore the fact that these terms are overwhelmingly one-sided and
non-negotiable, and even if we accept that it's meaningful for people to give consent
when sharing data about other people who may have also opted in — what is the recourse
for someone who has not opted into these systems at all? </p>
<p> Are those of us who have explicitly avoided agreeing to the Facebook terms of service
simply fair game for an industry-wide surveillance and targeting network? </p>
<h3>
<strong>Privilege</strong>
</h3>
<p> I don’t mean to critique people who have created a Facebook profile or suggest they deserve whatever they get. </p>
<p> My ability to avoid Facebook comes from privilege — I have existing social contacts with whom I know how to stay in touch without using Facebook's network. My job does not require that I use Facebook. I can afford the time and expense to communicate with my electoral representatives and political allies via other channels. </p>
<p> Many people do not have these privileges and are compelled to "opt in" on Facebook's non-negotiable terms. </p>
<p> Many journalists, organizers, schools, politicians, and others who have good reasons to oppose Facebook's centralized social control feel compelled by Facebook's reach and scale to participate in their practices, even those we know to be harmful. That includes the ACLU. </p>
<p> Privacy should not be a luxury good, and while I'm happy to encourage people to opt out of these subtle and socially fraught arrangements, I do not argue that anyone who has signed up has somehow relinquished concerns about their privacy. We need to evaluate privacy concerns in their full social contexts. These are not problems that can be resolved on an individual level, because of the interpersonal nature of much of this data and the complexities of the tradeoffs involved. </p>
<p> I don’t mean to critique people who have created a Facebook profile or suggest they
deserve whatever they get. </p>
<p> My ability to avoid Facebook comes from privilege — I have existing social contacts with
whom I know how to stay in touch without using Facebook's network. My job does not
require that I use Facebook. I can afford the time and expense to communicate with my
electoral representatives and political allies via other channels. </p>
<p> Many people do not have these privileges and are compelled to "opt in" on Facebook's
non-negotiable terms. </p>
<p> Many journalists, organizers, schools, politicians, and others who have good reasons to
oppose Facebook's centralized social control feel compelled by Facebook's reach and
scale to participate in their practices, even those we know to be harmful. That includes
the ACLU. </p>
<p> Privacy should not be a luxury good, and while I'm happy to encourage people to opt out
of these subtle and socially fraught arrangements, I do not argue that anyone who has
signed up has somehow relinquished concerns about their privacy. We need to evaluate
privacy concerns in their full social contexts. These are not problems that can be
resolved on an individual level, because of the interpersonal nature of much of this
data and the complexities of the tradeoffs involved. </p>
<h3>
<strong>Technical countermeasures</strong>
</h3>
<p> While they may not solve the problem, there are some technical steps people can take to limit the scope of these surveillance practices. For example, some web browsers do not send "third-party cookies" by default, or <a href="https://wiki.mozilla.org/Thirdparty">they scope cookies</a> so that centralized surveillance doesn't get a single view of one user. The most privacy-preserving modern browser is <a href="https://www.torproject.org/">the Tor Browser</a>, which everyone should have installed and available, even if it's not the browser they choose to use every day. It limits the surveillance ability of systems that you have not signed up for to track you as you move around the web. </p>
<p> You can also modify some browsers — for example, with plug-ins for <a href="https://requestpolicycontinued.github.io/">Firefox</a> and <a href="https://chrome.google.com/webstore/detail/umatrix/ogfcmafjalglgifnmanfmnieipoejdcf">Chrome</a> — so that they <a href="https://addons.mozilla.org/en-US/firefox/addon/umatrix/">do not send third-party</a> <a href="https://requestpolicycontinued.github.io/">requests at all</a>. Firefox is also exploring even more <a href="https://addons.mozilla.org/en-US/firefox/addon/multi-account-containers/">privacy-preserving techniques</a><a href="https://addons.mozilla.org/en-US/firefox/addon/multi-account-containers/">.</a>
<p> While they may not solve the problem, there are some technical steps people can take to
limit the scope of these surveillance practices. For example, some web browsers do not
send "third-party cookies" by default, or <a href="https://wiki.mozilla.org/Thirdparty">they
scope cookies</a> so that centralized surveillance doesn't get a single view of one
user. The most privacy-preserving modern browser is <a
href="https://www.torproject.org/">the Tor Browser</a>, which everyone should
have installed and available, even if it's not the browser they choose to use every day.
It limits the surveillance ability of systems that you have not signed up for to track
you as you move around the web. </p>
<p> You can also modify some browsers — for example, with plug-ins for <a
href="https://requestpolicycontinued.github.io/">Firefox</a> and <a
href="https://chrome.google.com/webstore/detail/umatrix/ogfcmafjalglgifnmanfmnieipoejdcf">Chrome</a>
— so that they <a href="https://addons.mozilla.org/en-US/firefox/addon/umatrix/">do not
send third-party</a> <a href="https://requestpolicycontinued.github.io/">requests at
all</a>. Firefox is also exploring even more <a
href="https://addons.mozilla.org/en-US/firefox/addon/multi-account-containers/">privacy-preserving
techniques</a><a
href="https://addons.mozilla.org/en-US/firefox/addon/multi-account-containers/">.</a>
</p>
<p> It can’t be denied, though, that these tools are harder to use than the web browsers most people are accustomed to, and they create barriers to some online activities. (For example, logging in to <a href="https://offcampushousing.uconn.edu/login">some sites</a> and accessing some <a href="https://filestore.community.support.microsoft.com/api/images/0253d8fb-b050-401a-834d-9d80a99c0b12">web applications</a> is impossible without third-party cookies.) </p>
<p> Some website operators take their visitors' privacy more seriously than others, by reducing the amount of third-party requests. For example, it's possible to display "share on Facebook" or "Like" buttons without sending user requests to Facebook in the first place. The ACLU's own website does this because we believe that the right to read with privacy is a fundamental protection for civic discourse. </p>
<p> If you are responsible for running a website, try browsing it with a third-party-blocking extension turned on. Think about how much information you're requiring your users to send to third parties as a condition for using your site. If you care about being a good steward of your visitors' data, you can re-design your website to reduce this kind of leakage. </p>
<p> It can’t be denied, though, that these tools are harder to use than the web browsers
most people are accustomed to, and they create barriers to some online activities. (For
example, logging in to <a href="https://offcampushousing.uconn.edu/login">some sites</a>
and accessing some <a
href="https://filestore.community.support.microsoft.com/api/images/0253d8fb-b050-401a-834d-9d80a99c0b12">web
applications</a> is impossible without third-party cookies.) </p>
<p> Some website operators take their visitors' privacy more seriously than others, by
reducing the amount of third-party requests. For example, it's possible to display
"share on Facebook" or "Like" buttons without sending user requests to Facebook in the
first place. The ACLU's own website does this because we believe that the right to read
with privacy is a fundamental protection for civic discourse. </p>
<p> If you are responsible for running a website, try browsing it with a
third-party-blocking extension turned on. Think about how much information you're
requiring your users to send to third parties as a condition for using your site. If you
care about being a good steward of your visitors' data, you can re-design your website
to reduce this kind of leakage. </p>
<h3>
<strong>Opting out?</strong>
</h3>
<p> Some advertisers claim that you can "opt out" of their targeted advertising, and even offer <a href="http://optout.aboutads.info/">a centralized place meant to help you do so</a>.&nbsp;However, my experience with these tools isn't a positive one. They don't appear to work all of the time. (In a recent experiment I conducted, two advertisers’ opt-out mechanisms failed to take effect.) And while advertisers claim to allow the user to opt out of "interest-based ads," it's not clear that the opt-outs govern data collection itself, rather than just the use of the collected data for displaying ads. Moreover, opting out on their terms requires the use of third-party cookies, thereby enabling another mechanism that other advertisers can then exploit. </p>
<p> It's also not clear how they function over time: How frequently do I need to take these steps? Do they expire? How often should I check back to make sure I’m still opted out? I'd much prefer an approach requiring me to opt <em>in</em> to surveillance and targeting. </p>
<p> Some advertisers claim that you can "opt out" of their targeted advertising, and even
offer <a href="http://optout.aboutads.info/">a centralized place meant to help you do
so</a>.&nbsp;However, my experience with these tools isn't a positive one. They
don't appear to work all of the time. (In a recent experiment I conducted, two
advertisers’ opt-out mechanisms failed to take effect.) And while advertisers claim to
allow the user to opt out of "interest-based ads," it's not clear that the opt-outs
govern data collection itself, rather than just the use of the collected data for
displaying ads. Moreover, opting out on their terms requires the use of third-party
cookies, thereby enabling another mechanism that other advertisers can then exploit.
</p>
<p> It's also not clear how they function over time: How frequently do I need to take these
steps? Do they expire? How often should I check back to make sure I’m still opted out?
I'd much prefer an approach requiring me to opt <em>in</em> to surveillance and
targeting. </p>
<h3>
<strong>Fix the surveillance economy, not just Facebook</strong>
</h3>
<p> These are just a few of the mechanisms that enable online tracking. Facebook is just one culprit in this online "surveillance economy," albeit a massive one — the company owns <a href="https://www.instagram.com/">Instagram</a>, <a href="https://atlassolutions.com/">Atlas</a>, <a href="https://www.whatsapp.com/">WhatsApp</a>, and dozens of other internet and technology companies and services. But it’s not the only player in this space. Google’s business model also relies on this kind of surveillance, and there are dozens of smaller players as well. </p>
<p> As we work to address the fallout from the current storm around Facebook and Cambridge Analytica, we can't afford to lose sight of these larger mechanisms at play. Cambridge Analytica's failures and mistakes are inherent to Facebook's business model. We need to seriously challenge the social structures that encourage people to opt in to this kind of surveillance. At the same time, we also need to protect those of us who manage to opt out. </p>
<p> These are just a few of the mechanisms that enable online tracking. Facebook is just one
culprit in this online "surveillance economy," albeit a massive one — the company owns
<a href="https://www.instagram.com/">Instagram</a>, <a
href="https://atlassolutions.com/">Atlas</a>, <a
href="https://www.whatsapp.com/">WhatsApp</a>, and dozens of other internet and
technology companies and services. But it’s not the only player in this space. Google’s
business model also relies on this kind of surveillance, and there are dozens of smaller
players as well. </p>
<p> As we work to address the fallout from the current storm around Facebook and Cambridge
Analytica, we can't afford to lose sight of these larger mechanisms at play. Cambridge
Analytica's failures and mistakes are inherent to Facebook's business model. We need to
seriously challenge the social structures that encourage people to opt in to this kind
of surveillance. At the same time, we also need to protect those of us who manage to opt
out. </p>
</div>
</div>

3137
app/src/main/assets/readability/test/test-pages/aclu/source.html
View File

File diff suppressed because one or more lines are too long

381
app/src/main/assets/readability/test/test-pages/archive-of-our-own/expected.html
View File

@ -1,111 +1,320 @@
<div id="readability-page-1" class="page">
<div role="article" id="chapters">
<h3 id="work"> Chapter Text </h3>
<p> Izuku was struggling to understand how he had even managed to get here, seated before the archvillain of Japan with only a sense of dread to keep him company. All Might sat concealed in an observation room, of the firm opinion that he could only aggravate the prisoner and he sent Izuku off with a strained smile. A vague haze hovered over Izuku’s memory. It started with a simple conversation gone astray on a long drive home. </p>
<p> “So, who is All For One? Do we know anything about him beyond what you told me before? He’s been imprisoned for months now.” Izuku remembered asking All Might from the backseat of the car as Detective Tsukauchi leisurely drove along a sprawling highway. </p>
<p> Playing on the car radio was an aftermath report of a villain attack in downtown Tokyo. Izuku caught the phrase “liquid body” from the female reporter before Detective Tsukauchi changed the channel. </p>
<p> “Nope. Still nothing. No one really wants to speak to him,” All Might had replied brightly. “He gives off polite airs, but he’s a piece of work.” All Might’s mostly obstructed shoulders in the front seat shrugged. “Not much you can do with someone like him. Everything that comes out is a threat or taunt.” All Might carefully waved his hand in a circular motion towards the side of his head. </p>
<p> “No one’s even made it through a full interview with him, from what I’ve heard,” Detective Tsukauchi added from behind the wheel. “He plays mind games with them. The prison also has a “no recent events” policy on any discussions with him as well. Just in case he ends up with ideas or has some means of communicating. Given that people only want to ask him about current events, it doesn’t leave much to talk about.” </p>
<p> “Wait, they still don’t know what Quirks he has?” Izuku asked exasperatedly. “They can’t if there’s still an information block on visits.” </p>
<p> “Nope. We have no idea what he can do. They can run DNA tests, but it’s not like anyone apart from him even knows how his Quirk works. They could get matches with any number of people, but if they’re not in a database then we can’t cross-reference them anyway. Even if they run an analysis, the data doesn’t mean anything without the ability to interpret it,” All Might gestured with a skeletal finger. “It’s a waste of time after the initial tests were conducted. They weren’t game to MRI him either, given he’s definitely got a Quirk that creates metal components.” </p>
<p> “No one’s bothered to ask him anything about… anything?” Izuku asked, dumbfounded. “He must be around two-hundred years old and people can’t think of a single non-current affairs thing to ask him?” </p>
<p> In some ways it was unfathomable that they’d let a potential resource go to waste. On the other hand, said potential resource had blown up a city, murdered numerous people and terrorised Japan for over a century. At the very least. </p>
<p> “Well, I tried to ask him about Shigaraki, but he didn’t say much of anything really. Some garbage about you being too dependent on me and him letting Shigaraki run wild and how he just wanted to be the ultimate evil,” All Might shrugged again. “He spends too much time talking about nothing.” </p>
<p> Izuku was struggling to understand how he had even managed to get here, seated before
the archvillain of Japan with only a sense of dread to keep him company. All Might sat
concealed in an observation room, of the firm opinion that he could only aggravate the
prisoner and he sent Izuku off with a strained smile. A vague haze hovered over Izuku’s
memory. It started with a simple conversation gone astray on a long drive home. </p>
<p> “So, who is All For One? Do we know anything about him beyond what you told me before?
He’s been imprisoned for months now.” Izuku remembered asking All Might from the
backseat of the car as Detective Tsukauchi leisurely drove along a sprawling
highway. </p>
<p> Playing on the car radio was an aftermath report of a villain attack in downtown Tokyo.
Izuku caught the phrase “liquid body” from the female reporter before Detective
Tsukauchi changed the channel. </p>
<p> “Nope. Still nothing. No one really wants to speak to him,” All Might had replied
brightly. “He gives off polite airs, but he’s a piece of work.” All Might’s mostly
obstructed shoulders in the front seat shrugged. “Not much you can do with someone like
him. Everything that comes out is a threat or taunt.” All Might carefully waved his hand
in a circular motion towards the side of his head. </p>
<p> “No one’s even made it through a full interview with him, from what I’ve heard,”
Detective Tsukauchi added from behind the wheel. “He plays mind games with them. The
prison also has a “no recent events” policy on any discussions with him as well. Just in
case he ends up with ideas or has some means of communicating. Given that people only
want to ask him about current events, it doesn’t leave much to talk about.” </p>
<p> “Wait, they still don’t know what Quirks he has?” Izuku asked exasperatedly. “They can’t
if there’s still an information block on visits.” </p>
<p> “Nope. We have no idea what he can do. They can run DNA tests, but it’s not like anyone
apart from him even knows how his Quirk works. They could get matches with any number of
people, but if they’re not in a database then we can’t cross-reference them anyway. Even
if they run an analysis, the data doesn’t mean anything without the ability to interpret
it,” All Might gestured with a skeletal finger. “It’s a waste of time after the initial
tests were conducted. They weren’t game to MRI him either, given he’s definitely got a
Quirk that creates metal components.” </p>
<p> “No one’s bothered to ask him anything about… anything?” Izuku asked, dumbfounded. “He
must be around two-hundred years old and people can’t think of a single non-current
affairs thing to ask him?” </p>
<p> In some ways it was unfathomable that they’d let a potential resource go to waste. On
the other hand, said potential resource had blown up a city, murdered numerous people
and terrorised Japan for over a century. At the very least. </p>
<p> “Well, I tried to ask him about Shigaraki, but he didn’t say much of anything really.
Some garbage about you being too dependent on me and him letting Shigaraki run wild and
how he just wanted to be the ultimate evil,” All Might shrugged again. “He spends too
much time talking about nothing.” </p>
<p> Izuku shifted his head onto his arm. “But, that’s not really nothing, is it?” </p>
<p> “What do you mean?” Izuku had the feeling that All Might would have been looking at him with the <i>you’re about to do something stupid aren’t you</i> expression that was thankfully becoming less common. </p>
<p> “Well, he clearly doesn’t know anything about us, All Might, if he thinks that you’re just going to let go of me after not even two years of being taught. Maybe Shigaraki was dependent on adult figures, but I don’t even remember my dad and mum’s been busy working and keeping the house together. I’ve never had a lot of adult supervision before,” Izuku laughed nervously. “I had to find ways to keep myself entertained. If anything, I’m on the disobedient side of the scale.” All Might outright giggled. </p>
<p> “I’ll say, especially after what happened with Overhaul. I’m surprised your mother let you leave the dorms again after that.” </p>
<p> “What do you mean?” Izuku had the feeling that All Might would have been looking at him
with the <i>you’re about to do something stupid aren’t you</i> expression that was
thankfully becoming less common. </p>
<p> “Well, he clearly doesn’t know anything about us, All Might, if he thinks that you’re
just going to let go of me after not even two years of being taught. Maybe Shigaraki was
dependent on adult figures, but I don’t even remember my dad and mum’s been busy working
and keeping the house together. I’ve never had a lot of adult supervision before,” Izuku
laughed nervously. “I had to find ways to keep myself entertained. If anything, I’m on
the disobedient side of the scale.” All Might outright giggled. </p>
<p> “I’ll say, especially after what happened with Overhaul. I’m surprised your mother let
you leave the dorms again after that.” </p>
<p> “I’m surprised she didn’t withdraw and ground me until I was thirty.” </p>
<p> “Oh? That strict?” Tsukauchi asked. </p>
<p> “She has her moments,” Izuku smiled fondly. “Do you think she’d agree to me asking the archvillain of Japan about his Quirk?” Izuku asked, only partially joking. There was an itch at the back of his head, a feeling of something missing that poked and prodded at his senses. </p>
<p> All Might coughed and sprayed the dash with a fine red mist. “Absolutely not! I forbid it!” </p>
<p> “She has her moments,” Izuku smiled fondly. “Do you think she’d agree to me asking the
archvillain of Japan about his Quirk?” Izuku asked, only partially joking. There was an
itch at the back of his head, a feeling of something missing that poked and prodded at
his senses. </p>
<p> All Might coughed and sprayed the dash with a fine red mist. “Absolutely not! I forbid
it!” </p>
<p> “That’s exactly why I’m asking her and not you,” Izuku grinned from the backseat. </p>
<p> “He’s evil!” </p>
<p> “He’s ancient. You honestly don’t wonder about the sort of things someone with that life experience and Quirk would have run across to end up the way he did?” </p>
<p> “Nope, he made it perfectly clear that he always wanted to be the supreme evil,” All Might snipped through folded arms. </p>
<p> “Yeah, and I’ll just take his word for that, won’t I?” Izuku grinned. “If he does nothing but lie, then that’s probably one too, but there’s a grain of truth in there somewhere.” </p>
<p> “What would you even do? Harass him into telling you his life story?” All Might sighed. </p>
<p> “Not when I can kill him with kindness. Who knows, it might even be poisonous for him.” </p>
<p> “You’re explaining this to your mother. Teacher or not, I’m not being on the receiving end of this one.” </p>
<p> “He’s ancient. You honestly don’t wonder about the sort of things someone with that life
experience and Quirk would have run across to end up the way he did?” </p>
<p> “Nope, he made it perfectly clear that he always wanted to be the supreme evil,” All
Might snipped through folded arms. </p>
<p> “Yeah, and I’ll just take his word for that, won’t I?” Izuku grinned. “If he does
nothing but lie, then that’s probably one too, but there’s a grain of truth in there
somewhere.” </p>
<p> “What would you even do? Harass him into telling you his life story?” All Might
sighed. </p>
<p> “Not when I can kill him with kindness. Who knows, it might even be poisonous for
him.” </p>
<p> “You’re explaining this to your mother. Teacher or not, I’m not being on the receiving
end of this one.” </p>
<p> Izuku blinked for a moment. “You’ll let me?” </p>
<p> “I’m not entirely for it, but any prospective information on what influenced Shigaraki can only be a good thing. If anything goes south we can pull you out pretty easily. Just be aware of who and what you’re dealing with.” Struggling, All Might turned a serious look to Izuku around the side of the seat. “<i>Only</i> if your mother gives the okay.” </p>
<p> The conversation turned to school for the rest of the way. </p>
<p> It might have been curiosity or it might have been the nagging sensation that chewed at his brain for the three weeks that he researched the subject of the conversation. All For One was a cryptid. Mystical in more ways than one, he was only a rumour on a network that was two-hundred years old. There were whispers of a shadowy figure who once ruled Japan, intermingled with a string of conspiracies and fragmented events. </p>
<p> Izuku had even braved the dark web, poking and prodding at some of the seedier elements of the world wide web. The internet had rumours, but the dark web had stories.<br />
<p> “I’m not entirely for it, but any prospective information on what influenced Shigaraki
can only be a good thing. If anything goes south we can pull you out pretty easily. Just
be aware of who and what you’re dealing with.” Struggling, All Might turned a serious
look to Izuku around the side of the seat. “<i>Only</i> if your mother gives the okay.”
</p>
<p> An implied yakuza wrote about his grandfather who lost a fire manipulation Quirk and his sanity without any reason. His grandfather had been institutionalised, crying and repeating “he took it, he took it” until his dying days. No one could console him. </p>
<p> Another user spoke of a nursing home where a room full of dementia residents inexplicably became docile and no longer used their Quirks on the increasingly disturbed staff. The nursing home erupted into flames just before a court case against them commenced. </p>
<p> A user with neon pink text spoke of how their great-great-great-great grandmother with a longevity Quirk had simply aged rapidly one day and passed away in her sleep, her face a mask of terror. No cause had ever been found. </p>
<p> A hacker provided a grainy CCTV recording of a heist and a scanned collection of documents from over a century ago, where there was a flash of light and entire bank vault had been emptied. What separated it from the usual robbery was that it contained a list containing confidential information on the Quirks of the First Generation. Izuku had greedily snavelled up and saved the video and documents to an external hard drive. </p>
<p> Paging through, Izuku saw someone recount how their Quirkless uncle had developed a warp Quirk and gone from rags to riches under a mysterious benefactor. A decade ago, the uncle had simply disappeared. </p>
<p> Numerous and terrifying, the stories were scattered nuggets of gold hidden across the web. They’d never last long, vanishing within hours of posting. Izuku bounced from proxy to proxy, fleeing from a series of deletions that seemed to follow Izuku’s aliased postings across snitch.ru, rabbit.az, aconspiracy.xfiles and their compatriots. </p>
<p> After thirty-two identity changes (all carefully logged in a separate notebook), a large amount of feigning communal interest in a lucky tabloid article on All For One which had been released at the start of the first of the three weeks, Izuku hung up his tinfoil hat and called it a month. He haphazardly tossed a bulging notebook into his bookshelf and lodged his hard drive in a gap containing seven others and went to dinner. </p>
<p> It took another week to present his research to All Might and Tsukauchi, whose jaws reached the proverbial floor. </p>
<p> “We never found any of this,” the Detective Tsukauchi exclaimed. “How did you find all of it?” </p>
<p> “I asked the right people. Turns out criminals have very long and very unforgiving memories,” Izuku explained through sunken eyes. “There’s more than this that could be linked to him, but these ones seem to be the most obvious.” </p>
<p> “They would do, you can’t be head of the underworld without making an army of enemies,” All Might agreed. “You know, if you can get any more information about these events, I think you’ll give people a lot of peace of mind.” </p>
<p> The conversation turned to school for the rest of the way. </p>
<p> It might have been curiosity or it might have been the nagging sensation that chewed at
his brain for the three weeks that he researched the subject of the conversation. All
For One was a cryptid. Mystical in more ways than one, he was only a rumour on a network
that was two-hundred years old. There were whispers of a shadowy figure who once ruled
Japan, intermingled with a string of conspiracies and fragmented events. </p>
<p> Izuku had even braved the dark web, poking and prodding at some of the seedier elements
of the world wide web. The internet had rumours, but the dark web had stories.<br/>
</p>
<p> An implied yakuza wrote about his grandfather who lost a fire manipulation Quirk and his
sanity without any reason. His grandfather had been institutionalised, crying and
repeating “he took it, he took it” until his dying days. No one could console him. </p>
<p> Another user spoke of a nursing home where a room full of dementia residents
inexplicably became docile and no longer used their Quirks on the increasingly disturbed
staff. The nursing home erupted into flames just before a court case against them
commenced. </p>
<p> A user with neon pink text spoke of how their great-great-great-great grandmother with a
longevity Quirk had simply aged rapidly one day and passed away in her sleep, her face a
mask of terror. No cause had ever been found. </p>
<p> A hacker provided a grainy CCTV recording of a heist and a scanned collection of
documents from over a century ago, where there was a flash of light and entire bank
vault had been emptied. What separated it from the usual robbery was that it contained a
list containing confidential information on the Quirks of the First Generation. Izuku
had greedily snavelled up and saved the video and documents to an external hard
drive. </p>
<p> Paging through, Izuku saw someone recount how their Quirkless uncle had developed a warp
Quirk and gone from rags to riches under a mysterious benefactor. A decade ago, the
uncle had simply disappeared. </p>
<p> Numerous and terrifying, the stories were scattered nuggets of gold hidden across the
web. They’d never last long, vanishing within hours of posting. Izuku bounced from proxy
to proxy, fleeing from a series of deletions that seemed to follow Izuku’s aliased
postings across snitch.ru, rabbit.az, aconspiracy.xfiles and their compatriots. </p>
<p> After thirty-two identity changes (all carefully logged in a separate notebook), a large
amount of feigning communal interest in a lucky tabloid article on All For One which had
been released at the start of the first of the three weeks, Izuku hung up his tinfoil
hat and called it a month. He haphazardly tossed a bulging notebook into his bookshelf
and lodged his hard drive in a gap containing seven others and went to dinner. </p>
<p> It took another week to present his research to All Might and Tsukauchi, whose jaws
reached the proverbial floor. </p>
<p> “We never found any of this,” the Detective Tsukauchi exclaimed. “How did you find all
of it?” </p>
<p> “I asked the right people. Turns out criminals have very long and very unforgiving
memories,” Izuku explained through sunken eyes. “There’s more than this that could be
linked to him, but these ones seem to be the most obvious.” </p>
<p> “They would do, you can’t be head of the underworld without making an army of enemies,”
All Might agreed. “You know, if you can get any more information about these events, I
think you’ll give people a lot of peace of mind.” </p>
<p> “Provided mum agrees to it.” </p>
<p> “Only if she agrees to it.” </p>
<p> It took another month to convince his mother, who eventually gave in once All Might provided an extremely comprehensive schedule of how the visitations and any resulting research would be carefully balanced against Izuku’s schoolwork and internship. </p>
<p> The day of the visit finally arrived, four months after the initial conversation, much to Izuku’s dismay. </p>
<p> Izuku remembered how he had arrived, with the Detective and All Might escorting him through its sterile, white innards. A list of rules rattled off at the gate, “no current affairs” was chief among them and an assertion that he’d be dragged from the room if need be if Izuku was to breach any of them. No smuggling of communication devices, no weapons, no Quirks, nothing that could compromise the prisoner’s secure status. </p>
<p> Heavily armoured and drilled guards leading him underground into the deepest bowels of the Tartarus complex. </p>
<p> Izuku understood the rules, dressed casually in a cotton t-shirt with “Shirt” printed across it in haphazard English and clutching at a carefully screened and utterly blank notebook. </p>
<p> Across from him, behind reinforced glass, the archvillain of Japan was bound and unmoving. </p>
<p> “Hello,” Izuku initiated uncertainly. His skin had been crawling the moment he crossed the threshold, a memory of the encounter and escape at the Kamino Ward months ago. </p>
<p> “Ah, All Might’s disciple,” drawled All For One, “is he too cowardly to come himself? Yet I don’t hear the garments of a hero.” With hardly a word out, All For One had already lunged for the figurative jugular. </p>
<p> A stray thought of <i>how does he know who I am if he’s blind and isn’t familiar with me?</i> whispered its way through Izuku’s head. </p>
<p> “Oh, no,” Izuku corrected hastily, almost relieved at the lack of any pretence, “I asked if I could talk to you. This isn’t exactly hero related.” </p>
<p> “I’m surprised he said yes.” While there was little by way of expression, Izuku could just about sense the contempt dripping from the prisoner’s tone. It wasn’t anything he wasn’t expecting. Kacchan had already said worse to him in earlier years. Water off a duck’s back. </p>
<p> “Well, he’s not my legal guardian, so I think you should be more surprised that mum said yes. She’s stricter with these things than All Might,” Izuku corrected again. “Mum gave the okay, but that was a stressful discussion.” And there it was, a miniscule twitch from the man opposite. A spasm more than anything else. <i>Interesting.</i> Pinned down as he was, the prisoner oozed irritation. </p>
<p> “At least your mother is a wise person. I wonder why the student doesn’t heed all of the advice of the teacher.” All For One’s tone didn’t indicate a question, so much as an implicit statement that All Might wasn’t worth listening to in any capacity. Kacchan would have hated the comparison, but the hostility had an almost comfortable familiarity. “He no doubt warned you off speaking to me, overprotective as he is, but here you are.” </p>
<p> Izuku found himself smiling at the thought of Kacchan’s outrage if he ever found out about the mental comparison as he replied. “I don’t think it’s normal for anyone my age to listen completely to their teachers. We pick and choose and run with what works best for us. He warned me, but I’m still here. Mum warned me as well, but I think she cared more about the time management aspect of it." </p>
<p> It took another month to convince his mother, who eventually gave in once All Might
provided an extremely comprehensive schedule of how the visitations and any resulting
research would be carefully balanced against Izuku’s schoolwork and internship. </p>
<p> The day of the visit finally arrived, four months after the initial conversation, much
to Izuku’s dismay. </p>
<p> Izuku remembered how he had arrived, with the Detective and All Might escorting him
through its sterile, white innards. A list of rules rattled off at the gate, “no current
affairs” was chief among them and an assertion that he’d be dragged from the room if
need be if Izuku was to breach any of them. No smuggling of communication devices, no
weapons, no Quirks, nothing that could compromise the prisoner’s secure status. </p>
<p> Heavily armoured and drilled guards leading him underground into the deepest bowels of
the Tartarus complex. </p>
<p> Izuku understood the rules, dressed casually in a cotton t-shirt with “Shirt” printed
across it in haphazard English and clutching at a carefully screened and utterly blank
notebook. </p>
<p> Across from him, behind reinforced glass, the archvillain of Japan was bound and
unmoving. </p>
<p> “Hello,” Izuku initiated uncertainly. His skin had been crawling the moment he crossed
the threshold, a memory of the encounter and escape at the Kamino Ward months ago. </p>
<p> “Ah, All Might’s disciple,” drawled All For One, “is he too cowardly to come himself?
Yet I don’t hear the garments of a hero.” With hardly a word out, All For One had
already lunged for the figurative jugular. </p>
<p> A stray thought of <i>how does he know who I am if he’s blind and isn’t familiar with
me?</i> whispered its way through Izuku’s head. </p>
<p> “Oh, no,” Izuku corrected hastily, almost relieved at the lack of any pretence, “I asked
if I could talk to you. This isn’t exactly hero related.” </p>
<p> “I’m surprised he said yes.” While there was little by way of expression, Izuku could
just about sense the contempt dripping from the prisoner’s tone. It wasn’t anything he
wasn’t expecting. Kacchan had already said worse to him in earlier years. Water off a
duck’s back. </p>
<p> “Well, he’s not my legal guardian, so I think you should be more surprised that mum said
yes. She’s stricter with these things than All Might,” Izuku corrected again. “Mum gave
the okay, but that was a stressful discussion.” And there it was, a miniscule twitch
from the man opposite. A spasm more than anything else. <i>Interesting.</i> Pinned down
as he was, the prisoner oozed irritation. </p>
<p> “At least your mother is a wise person. I wonder why the student doesn’t heed all of the
advice of the teacher.” All For One’s tone didn’t indicate a question, so much as an
implicit statement that All Might wasn’t worth listening to in any capacity. Kacchan
would have hated the comparison, but the hostility had an almost comfortable
familiarity. “He no doubt warned you off speaking to me, overprotective as he is, but
here you are.” </p>
<p> Izuku found himself smiling at the thought of Kacchan’s outrage if he ever found out
about the mental comparison as he replied. “I don’t think it’s normal for anyone my age
to listen completely to their teachers. We pick and choose and run with what works best
for us. He warned me, but I’m still here. Mum warned me as well, but I think she cared
more about the time management aspect of it." </p>
<p> “Is that a recent development?” All For One probed. </p>
<p> “Not really. My old homeroom teacher told me not to bother applying to U.A.” His mother’s beaming face had carried Izuku through the cheerful and resolute signing of that application form. </p>
<p> “Not really. My old homeroom teacher told me not to bother applying to U.A.” His
mother’s beaming face had carried Izuku through the cheerful and resolute signing of
that application form. </p>
<p> “I see you followed their advice to the letter,” came the snide, dismissive reply. </p>
<p> Izuku hoisted up his legs and sat cross-legged in his seat. Leaning slightly forward as he did so as to better prop up his notebook. </p>
<p> “You’re a walking contrarian, aren’t you? All Might told me about his run ins with you. What someone does or doesn’t do really doesn’t matter to you, you’ll just find a way to rationalise it as a negative and go on the attack anyway. What you’re currently doing is drawing attention away from yourself and focusing it on me so you can withhold information.” Izuku flipped open his notebook and put pen to paper. “You’ve got something fairly big to hide and you diverting attention exposes that motivation as existing anyway. The only real questions here are what and why?” Izuku paused in mortification as the man opposite’s lips parted. “I just said that aloud, didn’t I?” </p>
<p> Of the responses Izuku had expected, it wasn’t laughter. Unrestrained, Izuku would have expected a violent outburst. In this situation, he would have expected another scathing comment. Instead, All For One laughed breathily, leaning into his bonds. Wheezingly he spoke, “I’ll have to change tactics, if that one’s too transparent for you. How refreshing.” </p>
<p> Doing his best not to glow a blinding red and simultaneously pale at the interest, Izuku carried on. “I add it to the list when you do. I’m not emotionally involved enough to really be impacted by what you’re saying. I know about you in theory, but that’s it. Maybe All Might has a history with you, but I don’t really know enough about you personally to…” </p>
<p> “Care,” All For One supplied, somewhat subdued as he struggled to breathe. “You’re only here to satisfy your curiosity as to whether or not the stories were true.” </p>
<p> Izuku nodded, scratching at his notebook with his left hand. “Yes and no, I’m actually here to ask you about how your Quirk works.” <i>For now.</i>
<p> Izuku hoisted up his legs and sat cross-legged in his seat. Leaning slightly forward as
he did so as to better prop up his notebook. </p>
<p> “You’re a walking contrarian, aren’t you? All Might told me about his run ins with you.
What someone does or doesn’t do really doesn’t matter to you, you’ll just find a way to
rationalise it as a negative and go on the attack anyway. What you’re currently doing is
drawing attention away from yourself and focusing it on me so you can withhold
information.” Izuku flipped open his notebook and put pen to paper. “You’ve got
something fairly big to hide and you diverting attention exposes that motivation as
existing anyway. The only real questions here are what and why?” Izuku paused in
mortification as the man opposite’s lips parted. “I just said that aloud, didn’t
I?” </p>
<p> Of the responses Izuku had expected, it wasn’t laughter. Unrestrained, Izuku would have
expected a violent outburst. In this situation, he would have expected another scathing
comment. Instead, All For One laughed breathily, leaning into his bonds. Wheezingly he
spoke, “I’ll have to change tactics, if that one’s too transparent for you. How
refreshing.” </p>
<p> Doing his best not to glow a blinding red and simultaneously pale at the interest, Izuku
carried on. “I add it to the list when you do. I’m not emotionally involved enough to
really be impacted by what you’re saying. I know about you in theory, but that’s it.
Maybe All Might has a history with you, but I don’t really know enough about you
personally to…” </p>
<p> “Care,” All For One supplied, somewhat subdued as he struggled to breathe. “You’re only
here to satisfy your curiosity as to whether or not the stories were true.” </p>
<p> Izuku nodded, scratching at his notebook with his left hand. “Yes and no, I’m actually
here to ask you about how your Quirk works.” <i>For now.</i>
</p>
<p> Another chortle, more restrained that the last. </p>
<p> "What makes you think others haven’t already asked?” Had All For One been unrestrained, Izuku could imagine the stereotypical scene of the villain confidently leaning back in some overblown chair in a secret lair, drink of choice in hand, if the tone of voice was any indication. Deflections aside, the man easily rose to each comment. </p>
<p> “Whether or not they asked it’s irrelevant if they can’t read the answers.” Answers didn’t matter if the people involved were too attached to read into the answers. If none of the interviewers had managed a full interview, then it seemed unlikely that any sort of effort was put into understanding the villain. </p>
<p> “And you think you can? What expertise do you hold above theirs?” Doubt and reprimand weighted the words. Oddly enough, had Izuku been any younger he could have mistaken the man for a disapproving parent rebuking an overly ambitious child. Albeit an extremely evil one. </p>
<p> Izuku inhaled shortly and went for it. “If there’s something I know, it’s Quirks and how they work. Maybe I don’t know you, but I don’t really need to. Quirks fall under broad categories of function. You can take and give, consent doesn’t seem to be a factor. You either can’t “see” certain types of Quirks or you need to have prior knowledge of it before you take it with what I know about your brother. Despite your <i>nom de guerre</i>, because we both know it’s not your real name, you have a history of giving multiple Quirks and causing brain damage to the receiver. You clearly aren’t impacted by those same restrictions, so it must either alter your brain mapping or adjust functions to allow for simultaneous use and storage. It also must isolate or categories the Quirks you stock, because from the few people who do remember you, you creating certain Quirks is always in the context of giving them to someone else meaning there’s probably an inherent immunity to stop it from tainting your own Quirk with a mutation,” Izuku mumbled, almost to himself. “The only thing really in question about your Quirk is the finer details and whether or not you need to maintain those features or if they’re inherent and your hard limit for holding Quirks.” </p>
<p> There was silence, for only a moment. “If only my hands were free, I would clap for such a thoughtful assessment. Clearly you’re not all brawn,” All For One positively purred. “Speculate away.” A wide and slightly unhinged smile was directed at Izuku. </p>
<p> It was all Izuku could do not to wince at the eagerness. An image of a nervous All Might, hidden in the observation room above with the grim-faced prison staff, came to mind. </p>
<p> “I note that you said thoughtful and not correct,” and Izuku breathed and unsteadily jotted it down in his notebook. “You don’t seem bothered by the guess.” </p>
<p> “Few people live long enough to question my Quirk, let alone have the talent to guess so thoughtfully at its functions. It seems we share a hobby.” There was something terribly keen in that voice that hadn’t been there before, twisting itself through the compliment. </p>
<p> “I suppose it helps that you’re playing along out of boredom,” Izuku verbally dodged, unease uncoiling itself from the back of his mind. </p>
<p> “I <i>was</i> playing along out of boredom,” All For One corrected smoothly. “Now, I’m curious. Admittedly, my prior assumptions of you weren’t generous, but I’ve been too hasty in my assessments before.” </p>
<p> “I’ll pack up and leave now if that’s the case,” Izuku replied with only half an ear on the conversation as the words on his page began to drastically expand to distract himself from the building anxiety. </p>
<p> “Sarcasm, so you do have characteristics of a normal teenager. Your willingness to maim yourself has often left me wondering…” </p>
<p> “You’re deflecting again,” Izuku observed. “I’m not sure if that’s a nervous habit for you or if you’re doing it because I’m close to being right about your Quirk. That being said, I don’t think you know what a normal teenager is if Shigaraki is any indication. He’s about seven years too late for his rebellious phase.” </p>
<p> "What makes you think others haven’t already asked?” Had All For One been unrestrained,
Izuku could imagine the stereotypical scene of the villain confidently leaning back in
some overblown chair in a secret lair, drink of choice in hand, if the tone of voice was
any indication. Deflections aside, the man easily rose to each comment. </p>
<p> “Whether or not they asked it’s irrelevant if they can’t read the answers.” Answers
didn’t matter if the people involved were too attached to read into the answers. If none
of the interviewers had managed a full interview, then it seemed unlikely that any sort
of effort was put into understanding the villain. </p>
<p> “And you think you can? What expertise do you hold above theirs?” Doubt and reprimand
weighted the words. Oddly enough, had Izuku been any younger he could have mistaken the
man for a disapproving parent rebuking an overly ambitious child. Albeit an extremely
evil one. </p>
<p> Izuku inhaled shortly and went for it. “If there’s something I know, it’s Quirks and how
they work. Maybe I don’t know you, but I don’t really need to. Quirks fall under broad
categories of function. You can take and give, consent doesn’t seem to be a factor. You
either can’t “see” certain types of Quirks or you need to have prior knowledge of it
before you take it with what I know about your brother. Despite your <i>nom de
guerre</i>, because we both know it’s not your real name, you have a history of
giving multiple Quirks and causing brain damage to the receiver. You clearly aren’t
impacted by those same restrictions, so it must either alter your brain mapping or
adjust functions to allow for simultaneous use and storage. It also must isolate or
categories the Quirks you stock, because from the few people who do remember you, you
creating certain Quirks is always in the context of giving them to someone else meaning
there’s probably an inherent immunity to stop it from tainting your own Quirk with a
mutation,” Izuku mumbled, almost to himself. “The only thing really in question about
your Quirk is the finer details and whether or not you need to maintain those features
or if they’re inherent and your hard limit for holding Quirks.” </p>
<p> There was silence, for only a moment. “If only my hands were free, I would clap for such
a thoughtful assessment. Clearly you’re not all brawn,” All For One positively purred.
“Speculate away.” A wide and slightly unhinged smile was directed at Izuku. </p>
<p> It was all Izuku could do not to wince at the eagerness. An image of a nervous All
Might, hidden in the observation room above with the grim-faced prison staff, came to
mind. </p>
<p> “I note that you said thoughtful and not correct,” and Izuku breathed and unsteadily
jotted it down in his notebook. “You don’t seem bothered by the guess.” </p>
<p> “Few people live long enough to question my Quirk, let alone have the talent to guess so
thoughtfully at its functions. It seems we share a hobby.” There was something terribly
keen in that voice that hadn’t been there before, twisting itself through the
compliment. </p>
<p> “I suppose it helps that you’re playing along out of boredom,” Izuku verbally dodged,
unease uncoiling itself from the back of his mind. </p>
<p> “I <i>was</i> playing along out of boredom,” All For One corrected smoothly. “Now, I’m
curious. Admittedly, my prior assumptions of you weren’t generous, but I’ve been too
hasty in my assessments before.” </p>
<p> “I’ll pack up and leave now if that’s the case,” Izuku replied with only half an ear on
the conversation as the words on his page began to drastically expand to distract
himself from the building anxiety. </p>
<p> “Sarcasm, so you do have characteristics of a normal teenager. Your willingness to maim
yourself has often left me wondering…” </p>
<p> “You’re deflecting again,” Izuku observed. “I’m not sure if that’s a nervous habit for
you or if you’re doing it because I’m close to being right about your Quirk. That being
said, I don’t think you know what a normal teenager is if Shigaraki is any indication.
He’s about seven years too late for his rebellious phase.” </p>
<p> “I’m hurt and offended,” came the amused reply. </p>
<p> “By how Shigaraki ended up or your parenting? You only have yourself to blame for both of them.” </p>
<p> “How harsh. Shigaraki is a product of society that birthed him. I can’t take credit for all of the hard work,” All For One laid out invitingly. Perhaps someone else would have risen to the bait, but Izuku was already packing his mental bags and heading for the door. </p>
<p> Clearly the prisoner’s anticipation had registered poorly with someone in the observation room, because a voice rang through the air. “Time’s up Midoriya-kun.” </p>
<p> “Okay!” Izuku called back and etched out his last thoughtful of words, untangled his legs and rose to his feet. </p>
<p> “By how Shigaraki ended up or your parenting? You only have yourself to blame for both
of them.” </p>
<p> “How harsh. Shigaraki is a product of society that birthed him. I can’t take credit for
all of the hard work,” All For One laid out invitingly. Perhaps someone else would have
risen to the bait, but Izuku was already packing his mental bags and heading for the
door. </p>
<p> Clearly the prisoner’s anticipation had registered poorly with someone in the
observation room, because a voice rang through the air. “Time’s up Midoriya-kun.” </p>
<p> “Okay!” Izuku called back and etched out his last thoughtful of words, untangled his
legs and rose to his feet. </p>
<p> “What a shame, my visitations are always so short,” All For One spoke mournfully. </p>
<p> “Well, you did blow up half a city. They could have just let you suffocate instead. Same time next week, then?” Izuku offered brightly, notebook stuffed into a pocket and was followed out the door by wheezing laughter. </p>
<p> It was only after he had made it safely back to the communal room where All Might waited did he allow the spring to fade from his step and discard his nervous smile. Shuddering, he turned to All Might whose face was set in a grimace. </p>
<p> “I won’t say I told you so,” All Might offered, perched on the edge of his couch like a misshapen vulture. </p>
<p> “He’s… not really what I was expecting. I was expecting someone, more openly evil.” Izuku allowed himself to collapse into the leather of the seat. He shakily reached for the warm tea that had been clearly been prepared the moment Izuku left the cell. “I suppose he does it to lull people into a false sense of security. I didn’t understand how someone with only half a set of expressions could have “villain” written all over them until I met him.” </p>
<p> “He’s always been like that. He feigns concern and sympathy to lure in society’s outcasts. They’re easy targets,” All Might said through a mouthful of biscuit. </p>
<p> “Well, you did blow up half a city. They could have just let you suffocate instead. Same
time next week, then?” Izuku offered brightly, notebook stuffed into a pocket and was
followed out the door by wheezing laughter. </p>
<p> It was only after he had made it safely back to the communal room where All Might waited
did he allow the spring to fade from his step and discard his nervous smile. Shuddering,
he turned to All Might whose face was set in a grimace. </p>
<p> “I won’t say I told you so,” All Might offered, perched on the edge of his couch like a
misshapen vulture. </p>
<p> “He’s… not really what I was expecting. I was expecting someone, more openly evil.”
Izuku allowed himself to collapse into the leather of the seat. He shakily reached for
the warm tea that had been clearly been prepared the moment Izuku left the cell. “I
suppose he does it to lull people into a false sense of security. I didn’t understand
how someone with only half a set of expressions could have “villain” written all over
them until I met him.” </p>
<p> “He’s always been like that. He feigns concern and sympathy to lure in society’s
outcasts. They’re easy targets,” All Might said through a mouthful of biscuit. </p>
<p> “Has he ever tried it on any of the One For All successors?” </p>
<p> “Not really, but you might have accidentally given him the incentive for it. He never had access to any of the One For All wielders while they were young.” All Might snorted, “not that it’ll make a difference with you”. </p>
<p> “I think he was trying to gauge me for a world view before the wardens ended it. I need more time to work out his response to the stuff on his Quirk.” </p>
<p> “He’s conversation starved since it’s solitary confinement. If what the people monitoring his brain activity said was true, you’re the most exciting thing to have happened to him in months. He replied after you left, said he was looking forward to it.” </p>
<p> “Not really, but you might have accidentally given him the incentive for it. He never
had access to any of the One For All wielders while they were young.” All Might snorted,
“not that it’ll make a difference with you”. </p>
<p> “I think he was trying to gauge me for a world view before the wardens ended it. I need
more time to work out his response to the stuff on his Quirk.” </p>
<p> “He’s conversation starved since it’s solitary confinement. If what the people
monitoring his brain activity said was true, you’re the most exciting thing to have
happened to him in months. He replied after you left, said he was looking forward to
it.” </p>
<p> “That’s pretty sad." </p>
<p> “It’s even sadder that we’re the only two members of the public who have had anything to do with him. Stain gets a pile of mail from his “fans”, but All For One has nothing,” All Might waved a tea spoon. “That’s what he gets.” </p>
<p> “Let’s get out of here and tell Detective Tsukauchi how it went.” Izuku gulped down his tea and headed for the exit, with him and All Might reaching it at roughly the same amount of time. </p>
<p> “At least your mum’s making katsudon for us tonight," was All Might's only optimistic comment. </p>
<p> “It’s even sadder that we’re the only two members of the public who have had anything to
do with him. Stain gets a pile of mail from his “fans”, but All For One has nothing,”
All Might waved a tea spoon. “That’s what he gets.” </p>
<p> “Let’s get out of here and tell Detective Tsukauchi how it went.” Izuku gulped down his
tea and headed for the exit, with him and All Might reaching it at roughly the same
amount of time. </p>
<p> “At least your mum’s making katsudon for us tonight," was All Might's only optimistic
comment. </p>
<p> Anxiety was still ebbing over Izuku after Tsukauchi had been debriefed in the car. </p>
<p>
<i>“It seems we share a hobby.”</i> Haunted Izuku on the drive home. As if ripping someone’s Quirk from them and leaving them lying traumatised on the ground was just a fun pastime and not an act of grievous bodily harm.
<i>“It seems we share a hobby.”</i> Haunted Izuku on the drive home. As if ripping
someone’s Quirk from them and leaving them lying traumatised on the ground was just a
fun pastime and not an act of grievous bodily harm.
</p>
<p> And he’d be dealing with him again in another week. </p>
</div>

5382
app/src/main/assets/readability/test/test-pages/archive-of-our-own/source.html
View File

File diff suppressed because one or more lines are too long

74
app/src/main/assets/readability/test/test-pages/ars-1/expected.html
View File

@ -2,20 +2,43 @@
<div>
<header>
<h4> Biz &amp; IT — </h4>
<h2 itemprop="description"> Two-year-old bug exposes thousands of servers to crippling attack. </h2>
<h2 itemprop="description"> Two-year-old bug exposes thousands of servers to crippling
attack. </h2>
</header>
<div itemprop="articleBody">
<figure>
<img src="https://cdn.arstechnica.net/wp-content/uploads/2015/04/server-crash-640x426.jpg" alt="Just-released Minecraft exploit makes it easy to crash game servers" />
<img src="https://cdn.arstechnica.net/wp-content/uploads/2015/04/server-crash-640x426.jpg"
alt="Just-released Minecraft exploit makes it easy to crash game servers"/>
<figcaption class="caption">
</figcaption>
</figure>
<p> A flaw in the wildly popular online game <em>Minecraft</em> makes it easy for just about anyone to crash the server hosting the game, according to a computer programmer who has released proof-of-concept code that exploits the vulnerability. </p>
<p> "I thought a lot before writing this post," Pakistan-based developer Ammar Askar wrote in a <a href="http://blog.ammaraskar.com/minecraft-vulnerability-advisory">blog post published Thursday</a>, 21 months, he said, after privately reporting the bug to <em>Minecraft</em> developer Mojang. "On the one hand I don't want to expose thousands of servers to a major vulnerability, yet on the other hand Mojang has failed to act on it." </p>
<p> The bug resides in the <a href="https://github.com/ammaraskar/pyCraft">networking internals of the <em>Minecraft</em> protocol</a>. It allows the contents of inventory slots to be exchanged, so that, among other things, items in players' hotbars are displayed automatically after logging in. <em>Minecraft</em> items can also store arbitrary metadata in a file format known as <a href="http://wiki.vg/NBT">Named Binary Tag (NBT)</a>, which allows complex data structures to be kept in hierarchical nests. Askar has released <a href="https://github.com/ammaraskar/pyCraft/tree/nbt_exploit">proof-of-concept attack code</a> he said exploits the vulnerability to crash any server hosting the game. Here's how it works. </p>
<p> A flaw in the wildly popular online game <em>Minecraft</em> makes it easy for just
about anyone to crash the server hosting the game, according to a computer
programmer who has released proof-of-concept code that exploits the vulnerability.
</p>
<p> "I thought a lot before writing this post," Pakistan-based developer Ammar Askar
wrote in a <a href="http://blog.ammaraskar.com/minecraft-vulnerability-advisory">blog
post published Thursday</a>, 21 months, he said, after privately reporting the
bug to <em>Minecraft</em> developer Mojang. "On the one hand I don't want to expose
thousands of servers to a major vulnerability, yet on the other hand Mojang has
failed to act on it." </p>
<p> The bug resides in the <a href="https://github.com/ammaraskar/pyCraft">networking
internals of the <em>Minecraft</em> protocol</a>. It allows the contents of
inventory slots to be exchanged, so that, among other things, items in players'
hotbars are displayed automatically after logging in. <em>Minecraft</em> items can
also store arbitrary metadata in a file format known as <a
href="http://wiki.vg/NBT">Named Binary Tag (NBT)</a>, which allows complex
data structures to be kept in hierarchical nests. Askar has released <a
href="https://github.com/ammaraskar/pyCraft/tree/nbt_exploit">proof-of-concept
attack code</a> he said exploits the vulnerability to crash any server hosting
the game. Here's how it works. </p>
<blockquote>
<p> The vulnerability stems from the fact that the client is allowed to send the server information about certain slots. This, coupled with the NBT format’s nesting allows us to <em>craft</em> a packet that is incredibly complex for the server to deserialize but trivial for us to generate. </p>
<p> In my case, I chose to create lists within lists, down to five levels. This is a json representation of what it looks like. </p>
<p> The vulnerability stems from the fact that the client is allowed to send the
server information about certain slots. This, coupled with the NBT format’s
nesting allows us to <em>craft</em> a packet that is incredibly complex for the
server to deserialize but trivial for us to generate. </p>
<p> In my case, I chose to create lists within lists, down to five levels. This is a
json representation of what it looks like. </p>
<div>
<pre><code data-lang="javascript"><span>rekt</span><span>:</span> <span>{</span>
<span>list</span><span>:</span> <span>[</span>
@ -44,15 +67,38 @@
<span>...</span>
<span>}</span></code></pre>
</div>
<p> The root of the object, <code>rekt</code>, contains 300 lists. Each list has a list with 10 sublists, and each of those sublists has 10 of their own, up until 5 levels of recursion. That’s a total of <code>10^5 * 300 = 30,000,000</code> lists. </p>
<p> And this isn’t even the theoretical maximum for this attack. Just the nbt data for this payload is 26.6 megabytes. But luckily Minecraft implements a way to compress large packets, lucky us! zlib shrinks down our evil data to a mere 39 kilobytes. </p>
<p> Note: in previous versions of Minecraft, there was no protocol wide compression for big packets. Previously, NBT was sent compressed with gzip and prefixed with a signed short of its length, which reduced our maximum payload size to <code>2^15 - 1</code>. Now that the length is a varint capable of storing integers up to <code>2^28</code>, our potential for attack has increased significantly. </p>
<p> When the server will decompress our data, it’ll have 27 megs in a buffer somewhere in memory, but that isn’t the bit that’ll kill it. When it attempts to parse it into NBT, it’ll create java representations of the objects meaning suddenly, the sever is having to create several million java objects including ArrayLists. This runs the server out of memory and causes tremendous CPU load. </p>
<p> This vulnerability exists on almost all previous and current Minecraft versions as of 1.8.3, the packets used as attack vectors are the <a href="http://wiki.vg/Protocol#Player_Block_Placement">0x08: Block Placement Packet</a> and <a href="http://wiki.vg/Protocol#Creative_Inventory_Action">0x10: Creative Inventory Action</a>. </p>
<p> The fix for this vulnerability isn’t exactly that hard, the client should never really send a data structure as complex as NBT of arbitrary size and if it must, some form of recursion and size limits should be implemented. </p>
<p> The root of the object, <code>rekt</code>, contains 300 lists. Each list has a
list with 10 sublists, and each of those sublists has 10 of their own, up until
5 levels of recursion. That’s a total of <code>10^5 * 300 = 30,000,000</code>
lists. </p>
<p> And this isn’t even the theoretical maximum for this attack. Just the nbt data
for this payload is 26.6 megabytes. But luckily Minecraft implements a way to
compress large packets, lucky us! zlib shrinks down our evil data to a mere 39
kilobytes. </p>
<p> Note: in previous versions of Minecraft, there was no protocol wide compression
for big packets. Previously, NBT was sent compressed with gzip and prefixed with
a signed short of its length, which reduced our maximum payload size to <code>2^15
- 1</code>. Now that the length is a varint capable of storing integers up
to <code>2^28</code>, our potential for attack has increased significantly. </p>
<p> When the server will decompress our data, it’ll have 27 megs in a buffer
somewhere in memory, but that isn’t the bit that’ll kill it. When it attempts to
parse it into NBT, it’ll create java representations of the objects meaning
suddenly, the sever is having to create several million java objects including
ArrayLists. This runs the server out of memory and causes tremendous CPU
load. </p>
<p> This vulnerability exists on almost all previous and current Minecraft versions
as of 1.8.3, the packets used as attack vectors are the <a
href="http://wiki.vg/Protocol#Player_Block_Placement">0x08: Block
Placement Packet</a> and <a
href="http://wiki.vg/Protocol#Creative_Inventory_Action">0x10: Creative
Inventory Action</a>. </p>
<p> The fix for this vulnerability isn’t exactly that hard, the client should never
really send a data structure as complex as NBT of arbitrary size and if it must,
some form of recursion and size limits should be implemented. </p>
<p> These were the fixes that I recommended to Mojang 2 years ago. </p>
</blockquote>
<p> Ars is asking Mojang for comment and will update this post if company officials respond. </p>
<p> Ars is asking Mojang for comment and will update this post if company officials
respond. </p>
</div>
</div>
</div>

1252
app/src/main/assets/readability/test/test-pages/ars-1/source.html
View File

File diff suppressed because it is too large Load Diff

23
app/src/main/assets/readability/test/test-pages/base-url-base-element-relative/expected.html
View File

@ -1,7 +1,12 @@
<div id="readability-page-1" class="page">
<article>
<h2>Lorem</h2>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum. </p>
<p>Links</p>
<p><a href="http://fakehost/test/base/foo/bar/baz.html">link</a></p>
<p><a href="http://fakehost/test/base/foo/bar/baz.html">link</a></p>
@ -12,12 +17,16 @@
<p><a href="http://test/foo/bar/baz.html">link</a></p>
<p><a href="https://test/foo/bar/baz.html">link</a></p>
<p>Images</p>
<p><img src="http://fakehost/test/base/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/test/base/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/foo/bar/baz.png" /></p>
<p><img src="http://test/foo/bar/baz.png" /></p>
<p><img src="https://test/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/test/base/foo/bar/baz.png"/></p>
<p><img src="http://fakehost/test/base/foo/bar/baz.png"/></p>
<p><img src="http://fakehost/foo/bar/baz.png"/></p>
<p><img src="http://test/foo/bar/baz.png"/></p>
<p><img src="https://test/foo/bar/baz.png"/></p>
<h2>Foo</h2>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute
irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia
deserunt mollit anim id est laborum. </p>
</article>
</div>

32
app/src/main/assets/readability/test/test-pages/base-url-base-element-relative/source.html
View File

@ -1,20 +1,20 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<base href="base/"/>
<title>Base URL with base relative test</title>
<meta charset="utf-8"/>
<base href="base/"/>
<title>Base URL with base relative test</title>
</head>
<body>
<article>
<article>
<h1>Lorem</h1>
<div>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
<p>Links</p>
<p><a href="foo/bar/baz.html">link</a></p>
@ -33,12 +33,12 @@
<p><img src="https://test/foo/bar/baz.png"/></p>
<h2>Foo</h2>
<div>
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
</article>
</article>
</body>
</html>

23
app/src/main/assets/readability/test/test-pages/base-url-base-element/expected.html
View File

@ -1,7 +1,12 @@
<div id="readability-page-1" class="page">
<article>
<h2>Lorem</h2>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum. </p>
<p>Links</p>
<p><a href="http://fakehost/foo/bar/baz.html">link</a></p>
<p><a href="http://fakehost/foo/bar/baz.html">link</a></p>
@ -12,12 +17,16 @@
<p><a href="http://test/foo/bar/baz.html">link</a></p>
<p><a href="https://test/foo/bar/baz.html">link</a></p>
<p>Images</p>
<p><img src="http://fakehost/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/foo/bar/baz.png" /></p>
<p><img src="http://test/foo/bar/baz.png" /></p>
<p><img src="https://test/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/foo/bar/baz.png"/></p>
<p><img src="http://fakehost/foo/bar/baz.png"/></p>
<p><img src="http://fakehost/foo/bar/baz.png"/></p>
<p><img src="http://test/foo/bar/baz.png"/></p>
<p><img src="https://test/foo/bar/baz.png"/></p>
<h2>Foo</h2>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute
irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia
deserunt mollit anim id est laborum. </p>
</article>
</div>

32
app/src/main/assets/readability/test/test-pages/base-url-base-element/source.html
View File

@ -1,20 +1,20 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<base href="/"/>
<title>Base URL with base test</title>
<meta charset="utf-8"/>
<base href="/"/>
<title>Base URL with base test</title>
</head>
<body>
<article>
<article>
<h1>Lorem</h1>
<div>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
<p>Links</p>
<p><a href="foo/bar/baz.html">link</a></p>
@ -33,12 +33,12 @@
<p><img src="https://test/foo/bar/baz.png"/></p>
<h2>Foo</h2>
<div>
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
</article>
</article>
</body>
</html>

23
app/src/main/assets/readability/test/test-pages/base-url/expected.html
View File

@ -1,7 +1,12 @@
<div id="readability-page-1" class="page">
<article>
<h2>Lorem</h2>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum. </p>
<p>Links</p>
<p><a href="http://fakehost/test/foo/bar/baz.html">link</a></p>
<p><a href="http://fakehost/test/foo/bar/baz.html">link</a></p>
@ -12,12 +17,16 @@
<p><a href="http://test/foo/bar/baz.html">link</a></p>
<p><a href="https://test/foo/bar/baz.html">link</a></p>
<p>Images</p>
<p><img src="http://fakehost/test/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/test/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/foo/bar/baz.png" /></p>
<p><img src="http://test/foo/bar/baz.png" /></p>
<p><img src="https://test/foo/bar/baz.png" /></p>
<p><img src="http://fakehost/test/foo/bar/baz.png"/></p>
<p><img src="http://fakehost/test/foo/bar/baz.png"/></p>
<p><img src="http://fakehost/foo/bar/baz.png"/></p>
<p><img src="http://test/foo/bar/baz.png"/></p>
<p><img src="https://test/foo/bar/baz.png"/></p>
<h2>Foo</h2>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute
irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia
deserunt mollit anim id est laborum. </p>
</article>
</div>

30
app/src/main/assets/readability/test/test-pages/base-url/source.html
View File

@ -1,19 +1,19 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Base URL test</title>
<meta charset="utf-8"/>
<title>Base URL test</title>
</head>
<body>
<article>
<article>
<h1>Lorem</h1>
<div>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
<p>Links</p>
<p><a href="foo/bar/baz.html">link</a></p>
@ -32,12 +32,12 @@
<p><img src="https://test/foo/bar/baz.png"/></p>
<h2>Foo</h2>
<div>
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
</article>
</article>
</body>
</html>

17
app/src/main/assets/readability/test/test-pages/basic-tags-cleaning/expected.html
View File

@ -1,11 +1,18 @@
<div id="readability-page-1" class="page">
<div>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui
officia deserunt mollit anim id est laborum.</p>
</div>
<div>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui
officia deserunt mollit anim id est laborum.</p>
</div>
</div>

46
app/src/main/assets/readability/test/test-pages/basic-tags-cleaning/source.html
View File

@ -1,36 +1,36 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Basic tag cleaning test</title>
<meta charset="utf-8"/>
<title>Basic tag cleaning test</title>
</head>
<body>
<article>
<article>
<h1>Lorem</h1>
<div>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<iframe src="about:blank">Iframe fallback test</iframe>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<iframe src="about:blank">Iframe fallback test</iframe>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
</div>
<h2>Foo</h2>
<div>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<object data="foo.swf" type="application/x-shockwave-flash" width="88" height="31">
<param movie="foo.swf" />
</object>
<embed src="foo.swf"/>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<object data="foo.swf" type="application/x-shockwave-flash" width="88" height="31">
<param movie="foo.swf"/>
</object>
<embed src="foo.swf"/>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
</div>
</article>
</article>
</body>
</html>

130
app/src/main/assets/readability/test/test-pages/bbc-1/expected.html
View File

@ -1,51 +1,102 @@
<div id="readability-page-1" class="page">
<div property="articleBody">
<p>President Barack Obama has admitted that his failure to pass "common sense gun safety laws" in the US is the greatest frustration of his presidency. </p>
<p>In an interview with the BBC, Mr Obama said it was "distressing" not to have made progress on the issue "even in the face of repeated mass killings".</p>
<p>He vowed to keep trying, but the BBC's North America editor Jon Sopel said the president did not sound very confident. </p>
<p>President Barack Obama has admitted that his failure to pass "common sense gun safety
laws" in the US is the greatest frustration of his presidency. </p>
<p>In an interview with the BBC, Mr Obama said it was "distressing" not to have made
progress on the issue "even in the face of repeated mass killings".</p>
<p>He vowed to keep trying, but the BBC's North America editor Jon Sopel said the president
did not sound very confident. </p>
<p>However, Mr Obama said race relations had improved during his presidency. </p>
<p>Hours after the interview, a gunman opened fire at a cinema in the US state of Louisiana, killing two people and injuring several others before shooting himself.</p>
<p>Hours after the interview, a gunman opened fire at a cinema in the US state of Louisiana,
killing two people and injuring several others before shooting himself.</p>
<p>In a wide-ranging interview, President Obama also said:</p>
<ul>
<li>
<a href="http://www.bbc.co.uk/news/uk-politics-33647154">The UK must stay in the EU</a> to have influence on the world stage
<a href="http://www.bbc.co.uk/news/uk-politics-33647154">The UK must stay in the
EU</a> to have influence on the world stage
</li>
<li>He is confident the Iran nuclear deal will be passed by Congress </li>
<li>He is confident the Iran nuclear deal will be passed by Congress</li>
<li>Syria needs a political solution in order to defeat the Islamic State group</li>
<li>He would speak "bluntly" against corruption <a href="http://www.bbc.co.uk/news/world-us-canada-33646563">and human rights violations in Kenya</a>
<li>He would speak "bluntly" against corruption <a
href="http://www.bbc.co.uk/news/world-us-canada-33646563">and human rights
violations in Kenya</a>
</li>
<li>He would defend his advocacy of gay rights following protests in Kenya</li>
<li>Despite racial tensions, the US is becoming more diverse and more tolerant</li>
</ul>
<p><a href="http://www.bbc.co.uk/news/world-us-canada-33646542">Read the full transcript of his interview</a></p>
<p>Mr Obama lands in Kenya later on Friday for his first visit since becoming president. </p>
<p>But with just 18 months left in power, he said gun control was the area where he has been "most frustrated and most stymied" since coming to power in 2009.</p>
<p>"If you look at the number of Americans killed since 9/11 by terrorism, it's less than 100. If you look at the number that have been killed by gun violence, it's in the tens of thousands," Mr Obama said. </p>
<figure><img src="http://ichef.bbci.co.uk/news/555/cpsprodpb/462D/production/_84456971_gettyimages-167501087.jpg" datasrc="http://ichef.bbci.co.uk/news/976/cpsprodpb/462D/production/_84456971_gettyimages-167501087.jpg" alt="Gun control campaigners protest in McPhearson Square in Washington DC - 25 April 2013" height="549" width="976" />
<p><a href="http://www.bbc.co.uk/news/world-us-canada-33646542">Read the full transcript of
his interview</a></p>
<p>Mr Obama lands in Kenya later on Friday for his first visit since becoming
president. </p>
<p>But with just 18 months left in power, he said gun control was the area where he has been
"most frustrated and most stymied" since coming to power in 2009.</p>
<p>"If you look at the number of Americans killed since 9/11 by terrorism, it's less than
100. If you look at the number that have been killed by gun violence, it's in the tens
of thousands," Mr Obama said. </p>
<figure><img
src="http://ichef.bbci.co.uk/news/555/cpsprodpb/462D/production/_84456971_gettyimages-167501087.jpg"
datasrc="http://ichef.bbci.co.uk/news/976/cpsprodpb/462D/production/_84456971_gettyimages-167501087.jpg"
alt="Gun control campaigners protest in McPhearson Square in Washington DC - 25 April 2013"
height="549" width="976"/>
<figcaption>
<span> The president said he would continue fighting for greater gun control laws </span>
</figcaption>
</figure>
<p>"For us not to be able to resolve that issue has been something that is distressing," he added. </p>
<p>Mr Obama has pushed for stricter gun control throughout his presidency but has been unable to secure any significant changes to the laws. </p>
<p>After nine African-American churchgoers were killed in South Carolina in June, he admitted "politics in this town" meant there were few options available.</p>
<figure><img src="http://ichef.bbci.co.uk/news/555/media/images/76020000/jpg/_76020974_line976.jpg" datasrc="http://ichef.bbci.co.uk/news/464/media/images/76020000/jpg/_76020974_line976.jpg" alt="line" height="2" width="464" /></figure>
<p>"For us not to be able to resolve that issue has been something that is distressing," he
added. </p>
<p>Mr Obama has pushed for stricter gun control throughout his presidency but has been
unable to secure any significant changes to the laws. </p>
<p>After nine African-American churchgoers were killed in South Carolina in June, he
admitted "politics in this town" meant there were few options available.</p>
<figure><img
src="http://ichef.bbci.co.uk/news/555/media/images/76020000/jpg/_76020974_line976.jpg"
datasrc="http://ichef.bbci.co.uk/news/464/media/images/76020000/jpg/_76020974_line976.jpg"
alt="line" height="2" width="464"/></figure>
<h2>Analysis: Jon Sopel, BBC News, Washington</h2>
<figure><img src="http://ichef-1.bbci.co.uk/news/555/cpsprodpb/6D3D/production/_84456972_p072315al-0500.jpg" datasrc="http://ichef-1.bbci.co.uk/news/976/cpsprodpb/6D3D/production/_84456972_p072315al-0500.jpg" alt="President Barack Obama participates in an interview with Jon Sopel of BBC in the Roosevelt Room of the White House - 23 July 2015" height="549" width="976" /></figure>
<p>Nine months ago, the president seemed like a spent force, after taking a beating in the midterm elections, during which members of his own party were reluctant to campaign on his record. </p>
<p>But the man sat before me today was relaxed and confident, buoyed by a string of "wins" on healthcare, Cuba and Iran, after bitter and ongoing battles with his many critics. </p>
<p>The only body swerve the president performed was when I asked him <a href="http://www.bbc.co.uk/news/world-us-canada-33643168"> how many minds he had changed on the Iran nuclear deal </a>after an intense sell aimed at Gulf allies and members of US Congress who remain implacably opposed. </p>
<p>There was a momentary flicker across the president's face as if to say "You think you got me?" before his smile returned and he proceeded to talk about how Congress would come round.</p>
<p>But notably, he did not give a direct answer to that question, which leaves me with the impression that he has persuaded precisely zero.</p>
<p><a href="http://www.bbc.co.uk/news/world-us-canada-33646875">Five things we learned from Obama interview</a></p>
<p><a href="http://www.bbc.co.uk/news/world-us-canada-33646545">The presidential body swerve</a></p>
<figure><img src="http://ichef.bbci.co.uk/news/555/media/images/76020000/jpg/_76020974_line976.jpg" datasrc="http://ichef.bbci.co.uk/news/464/media/images/76020000/jpg/_76020974_line976.jpg" alt="line" height="2" width="464" /></figure>
<p>On race relations, Mr Obama said recent concerns around policing and mass incarcerations were "legitimate and deserve intense attention" but insisted progress had been made. </p>
<p>Children growing up during the eight years of his presidency "will have a different view of race relations in this country and what's possible," he said. </p>
<p>"There are going to be tensions that arise. But if you look at my daughters' generation, they have an attitude about race that's entirely different than even my generation."</p>
<p>Talking about how he was feeling after his recent successes, he said "every president, every leader has strengths and weaknesses". </p>
<p>"One of my strengths is I have a pretty even temperament. I don't get too high when it's high and I don't get too low when it's low," he said. </p>
<figure><img src="http://ichef-1.bbci.co.uk/news/555/cpsprodpb/142FD/production/_84458628_shirtreuters.jpg" datasrc="http://ichef-1.bbci.co.uk/news/976/cpsprodpb/142FD/production/_84458628_shirtreuters.jpg" alt="Customer looks at Obama shirts at a stall in Nairobi&apos;s Kibera slums, 23 July 2015" height="549" width="976" />
<figure><img
src="http://ichef-1.bbci.co.uk/news/555/cpsprodpb/6D3D/production/_84456972_p072315al-0500.jpg"
datasrc="http://ichef-1.bbci.co.uk/news/976/cpsprodpb/6D3D/production/_84456972_p072315al-0500.jpg"
alt="President Barack Obama participates in an interview with Jon Sopel of BBC in the Roosevelt Room of the White House - 23 July 2015"
height="549" width="976"/></figure>
<p>Nine months ago, the president seemed like a spent force, after taking a beating in the
midterm elections, during which members of his own party were reluctant to campaign on
his record. </p>
<p>But the man sat before me today was relaxed and confident, buoyed by a string of "wins"
on healthcare, Cuba and Iran, after bitter and ongoing battles with his many
critics. </p>
<p>The only body swerve the president performed was when I asked him <a
href="http://www.bbc.co.uk/news/world-us-canada-33643168"> how many minds he had
changed on the Iran nuclear deal </a>after an intense sell aimed at Gulf allies and
members of US Congress who remain implacably opposed. </p>
<p>There was a momentary flicker across the president's face as if to say "You think you got
me?" before his smile returned and he proceeded to talk about how Congress would come
round.</p>
<p>But notably, he did not give a direct answer to that question, which leaves me with the
impression that he has persuaded precisely zero.</p>
<p><a href="http://www.bbc.co.uk/news/world-us-canada-33646875">Five things we learned from
Obama interview</a></p>
<p><a href="http://www.bbc.co.uk/news/world-us-canada-33646545">The presidential body
swerve</a></p>
<figure><img
src="http://ichef.bbci.co.uk/news/555/media/images/76020000/jpg/_76020974_line976.jpg"
datasrc="http://ichef.bbci.co.uk/news/464/media/images/76020000/jpg/_76020974_line976.jpg"
alt="line" height="2" width="464"/></figure>
<p>On race relations, Mr Obama said recent concerns around policing and mass incarcerations
were "legitimate and deserve intense attention" but insisted progress had been
made. </p>
<p>Children growing up during the eight years of his presidency "will have a different view
of race relations in this country and what's possible," he said. </p>
<p>"There are going to be tensions that arise. But if you look at my daughters' generation,
they have an attitude about race that's entirely different than even my generation."</p>
<p>Talking about how he was feeling after his recent successes, he said "every president,
every leader has strengths and weaknesses". </p>
<p>"One of my strengths is I have a pretty even temperament. I don't get too high when it's
high and I don't get too low when it's low," he said. </p>
<figure><img
src="http://ichef-1.bbci.co.uk/news/555/cpsprodpb/142FD/production/_84458628_shirtreuters.jpg"
datasrc="http://ichef-1.bbci.co.uk/news/976/cpsprodpb/142FD/production/_84458628_shirtreuters.jpg"
alt="Customer looks at Obama shirts at a stall in Nairobi&apos;s Kibera slums, 23 July 2015"
height="549" width="976"/>
<figcaption>
<span> Kenya is getting ready to welcome the US president </span>
</figcaption>
@ -53,10 +104,17 @@
<h2>Kenya trip</h2>
<p>Mr Obama was speaking to the BBC at the White House before departing for Kenya.</p>
<p>His father was Kenyan and the president is expected to meet relatives in Nairobi.</p>
<p>Mr Obama has faced criticism in the country after the US legalised gay marriage. However, in his interview, the president said he would not fall silent on the issue.</p>
<p>"I am not a fan of discrimination and bullying of anybody on the basis of race, on the basis of religion, on the basis of sexual orientation or gender," he said.</p>
<p>The president also admitted that some African governments, including Kenya's, needed to improve their records on human rights and democracy. However, he defended his decision to engage with and visit those governments. </p>
<p>"Well, they're not ideal institutions. But what we found is, is that when we combined blunt talk with engagement, that gives us the best opportunity to influence and open up space for civil society." </p>
<p>Mr Obama will become the first US president to address the African Union when he travels on to Ethiopia on Sunday.</p>
<p>Mr Obama has faced criticism in the country after the US legalised gay marriage. However,
in his interview, the president said he would not fall silent on the issue.</p>
<p>"I am not a fan of discrimination and bullying of anybody on the basis of race, on the
basis of religion, on the basis of sexual orientation or gender," he said.</p>
<p>The president also admitted that some African governments, including Kenya's, needed to
improve their records on human rights and democracy. However, he defended his decision
to engage with and visit those governments. </p>
<p>"Well, they're not ideal institutions. But what we found is, is that when we combined
blunt talk with engagement, that gives us the best opportunity to influence and open up
space for civil society." </p>
<p>Mr Obama will become the first US president to address the African Union when he travels
on to Ethiopia on Sunday.</p>
</div>
</div>

3929
app/src/main/assets/readability/test/test-pages/bbc-1/source.html
View File

File diff suppressed because one or more lines are too long

332
app/src/main/assets/readability/test/test-pages/blogger/expected.html
View File

@ -1,117 +1,285 @@
<div id="readability-page-1" class="page">
<div id="post-body-932306423056216142" itemprop="description articleBody">
<p> I've written a couple of posts in the past few months but they were all for <a href="http://blog.ioactive.com/search/label/Andrew%20Zonenberg">the blog at work</a> so I figured I'm long overdue for one on Silicon Exposed.</p>
<p> I've written a couple of posts in the past few months but they were all for <a
href="http://blog.ioactive.com/search/label/Andrew%20Zonenberg">the blog at work</a>
so I figured I'm long overdue for one on Silicon Exposed.</p>
<h2> So what's a GreenPak?</h2>
<p> Silego Technology is a fabless semiconductor company located in the SF Bay area, which makes (among other things) a line of programmable logic devices known as GreenPak. Their <a href="http://www.silego.com/products/greenpak5.html">5th generation parts</a> were just announced, but I started this project before that happened so I'm still targeting the <a href="http://www.silego.com/products/greenpak4.html">4th generation</a>.</p>
<p> GreenPak devices are kind of like itty bitty <a href="http://www.cypress.com/products/32-bit-arm-cortex-m-psoc">PSoCs</a> - they have a mixed signal fabric with an ADC, DACs, comparators, voltage references, plus a digital LUT/FF fabric and some typical digital MCU peripherals like counters and oscillators (but no CPU).</p>
<p> It's actually an interesting architecture - FPGAs (including some devices marketed as CPLDs) are a 2D array of LUTs connected via wires to adjacent cells, and true (product term) CPLDs are a star topology of AND-OR arrays connected by a crossbar. GreenPak, on the other hand, is a star topology of LUTs, flipflops, and analog/digital hard IP connected to a crossbar.</p>
<p> Without further ado, here's a block diagram showing all the cool stuff you get in the SLG46620V:</p>
<p> Silego Technology is a fabless semiconductor company located in the SF Bay area, which
makes (among other things) a line of programmable logic devices known as GreenPak. Their
<a href="http://www.silego.com/products/greenpak5.html">5th generation parts</a> were
just announced, but I started this project before that happened so I'm still targeting
the <a href="http://www.silego.com/products/greenpak4.html">4th generation</a>.</p>
<p> GreenPak devices are kind of like itty bitty <a
href="http://www.cypress.com/products/32-bit-arm-cortex-m-psoc">PSoCs</a> - they
have a mixed signal fabric with an ADC, DACs, comparators, voltage references, plus a
digital LUT/FF fabric and some typical digital MCU peripherals like counters and
oscillators (but no CPU).</p>
<p> It's actually an interesting architecture - FPGAs (including some devices marketed as
CPLDs) are a 2D array of LUTs connected via wires to adjacent cells, and true (product
term) CPLDs are a star topology of AND-OR arrays connected by a crossbar. GreenPak, on
the other hand, is a star topology of LUTs, flipflops, and analog/digital hard IP
connected to a crossbar.</p>
<p> Without further ado, here's a block diagram showing all the cool stuff you get in the
SLG46620V:</p>
<table>
<tbody>
<tr>
<td>
<a href="https://1.bp.blogspot.com/-YIPC5jkXkDE/Vy7YPSqFKWI/AAAAAAAAAxI/a7D6Ji2GxoUvcrwUkI4RLZcr2LFQEJCTACLcB/s1600/block-diagram.png" imageanchor="1"><img height="512" src="https://1.bp.blogspot.com/-YIPC5jkXkDE/Vy7YPSqFKWI/AAAAAAAAAxI/a7D6Ji2GxoUvcrwUkI4RLZcr2LFQEJCTACLcB/s640/block-diagram.png" width="640" /></a>
</td>
</tr>
<tr>
<td>SLG46620V block diagram (from device datasheet)</td>
</tr>
<tr>
<td>
<a href="https://1.bp.blogspot.com/-YIPC5jkXkDE/Vy7YPSqFKWI/AAAAAAAAAxI/a7D6Ji2GxoUvcrwUkI4RLZcr2LFQEJCTACLcB/s1600/block-diagram.png"
imageanchor="1"><img height="512"
src="https://1.bp.blogspot.com/-YIPC5jkXkDE/Vy7YPSqFKWI/AAAAAAAAAxI/a7D6Ji2GxoUvcrwUkI4RLZcr2LFQEJCTACLcB/s640/block-diagram.png"
width="640"/></a>
</td>
</tr>
<tr>
<td>SLG46620V block diagram (from device datasheet)</td>
</tr>
</tbody>
</table>
<p> They're also tiny (the SLG46620V is a 20-pin 0.4mm pitch STQFN measuring 2x3 mm, and the lower gate count SLG46140V is a mere 1.6x2 mm) and probably the cheapest programmable logic device on the market - $0.50 in low volume and less than $0.40 in larger quantities.</p>
<p> The Vdd range of GreenPak4 is huge, more like what you'd expect from an MCU than an FPGA! It can run on anything from 1.8 to 5V, although performance is only specified at 1.8, 3.3, and 5V nominal voltages. There's also a dual-rail version that trades one of the GPIO pins for a second power supply pin, allowing you to interface to logic at two different voltage levels.</p>
<p> To support low-cost/space-constrained applications, they even have the configuration memory on die. It's one-time programmable and needs external Vpp to program (presumably Silego didn't want to waste die area on charge pumps that would only be used once) but has a SRAM programming mode for prototyping.</p>
<p> The best part is that the development software (GreenPak Designer) is free of charge and provided for all major operating systems including Linux! Unfortunately, the only supported design entry method is schematic entry and there's no way to write your design in a HDL.</p>
<p> While schematics may be fine for quick tinkering on really simple designs, they quickly get unwieldy. The nightmare of a circuit shown below is just a bunch of counters hooked up to LEDs that blink at various rates.</p>
<p> They're also tiny (the SLG46620V is a 20-pin 0.4mm pitch STQFN measuring 2x3 mm, and the
lower gate count SLG46140V is a mere 1.6x2 mm) and probably the cheapest programmable
logic device on the market - $0.50 in low volume and less than $0.40 in larger
quantities.</p>
<p> The Vdd range of GreenPak4 is huge, more like what you'd expect from an MCU than an
FPGA! It can run on anything from 1.8 to 5V, although performance is only specified at
1.8, 3.3, and 5V nominal voltages. There's also a dual-rail version that trades one of
the GPIO pins for a second power supply pin, allowing you to interface to logic at two
different voltage levels.</p>
<p> To support low-cost/space-constrained applications, they even have the configuration
memory on die. It's one-time programmable and needs external Vpp to program (presumably
Silego didn't want to waste die area on charge pumps that would only be used once) but
has a SRAM programming mode for prototyping.</p>
<p> The best part is that the development software (GreenPak Designer) is free of charge and
provided for all major operating systems including Linux! Unfortunately, the only
supported design entry method is schematic entry and there's no way to write your design
in a HDL.</p>
<p> While schematics may be fine for quick tinkering on really simple designs, they quickly
get unwieldy. The nightmare of a circuit shown below is just a bunch of counters hooked
up to LEDs that blink at various rates.</p>
<table>
<tbody>
<tr>
<td>
<a href="https://1.bp.blogspot.com/-k3naUT3uXao/Vy7WFac246I/AAAAAAAAAw8/mePy_ostO8QJra5ZJrbP2WGhTlJ0B_r8gCLcB/s1600/schematic-from-hell.png" imageanchor="1"><img height="334" src="https://1.bp.blogspot.com/-k3naUT3uXao/Vy7WFac246I/AAAAAAAAAw8/mePy_ostO8QJra5ZJrbP2WGhTlJ0B_r8gCLcB/s640/schematic-from-hell.png" width="640" /></a>
</td>
</tr>
<tr>
<td>Schematic from hell!</td>
</tr>
<tr>
<td>
<a href="https://1.bp.blogspot.com/-k3naUT3uXao/Vy7WFac246I/AAAAAAAAAw8/mePy_ostO8QJra5ZJrbP2WGhTlJ0B_r8gCLcB/s1600/schematic-from-hell.png"
imageanchor="1"><img height="334"
src="https://1.bp.blogspot.com/-k3naUT3uXao/Vy7WFac246I/AAAAAAAAAw8/mePy_ostO8QJra5ZJrbP2WGhTlJ0B_r8gCLcB/s640/schematic-from-hell.png"
width="640"/></a>
</td>
</tr>
<tr>
<td>Schematic from hell!</td>
</tr>
</tbody>
</table>
<p> As if this wasn't enough of a problem, the largest GreenPak4 device (the SLG46620V) is split into two halves with limited routing between them, and the GUI doesn't help the user manage this complexity at all - you have to draw your schematic in two halves and add "cross connections" between them.</p>
<p> The icing on the cake is that schematics are a pain to diff and collaborate on. Although GreenPak schematics are XML based, which is a touch better than binary, who wants to read a giant XML diff and try to figure out what's going on in the circuit?</p>
<p> This isn't going to be a post on the quirks of Silego's software, though - that would be boring. As it turns out, there's one more exciting feature of these chips that I didn't mention earlier: the configuration bitstream is 100% documented in the device datasheet! This is unheard of in the programmable logic world. As Nick of Arachnid Labs <a href="http://www.arachnidlabs.com/blog/2015/03/30/greenpak/">says</a>, the chip is "just dying for someone to write a VHDL or Verilog compiler for it". As you can probably guess by from the title of this post, I've been busy doing exactly that.</p>
<p> As if this wasn't enough of a problem, the largest GreenPak4 device (the SLG46620V) is
split into two halves with limited routing between them, and the GUI doesn't help the
user manage this complexity at all - you have to draw your schematic in two halves and
add "cross connections" between them.</p>
<p> The icing on the cake is that schematics are a pain to diff and collaborate on. Although
GreenPak schematics are XML based, which is a touch better than binary, who wants to
read a giant XML diff and try to figure out what's going on in the circuit?</p>
<p> This isn't going to be a post on the quirks of Silego's software, though - that would be
boring. As it turns out, there's one more exciting feature of these chips that I didn't
mention earlier: the configuration bitstream is 100% documented in the device datasheet!
This is unheard of in the programmable logic world. As Nick of Arachnid Labs <a
href="http://www.arachnidlabs.com/blog/2015/03/30/greenpak/">says</a>, the chip
is "just dying for someone to write a VHDL or Verilog compiler for it". As you can
probably guess by from the title of this post, I've been busy doing exactly that.</p>
<h2> Great! How does it work?</h2>
<p> Rather than wasting time writing a synthesizer, I decided to write a GreenPak technology library for Clifford Wolf's excellent open source synthesis tool, <a href="http://www.clifford.at/yosys/">Yosys</a>, and then make a place-and-route tool to turn that into a final netlist. The post-PAR netlist can then be loaded into GreenPak Designer in order to program the device.</p>
<p> The first step of the process is to run the "synth_greenpak4" Yosys flow on the Verilog source. This runs a generic RTL synthesis pass, then some coarse-grained extraction passes to infer shift register and counter cells from behavioral logic, and finally maps the remaining logic to LUT/FF cells and outputs a JSON-formatted netlist.</p>
<p> Once the design has been synthesized, my tool (named, surprisingly, gp4par) is then launched on the netlist. It begins by parsing the JSON and constructing a directed graph of cell objects in memory. A second graph, containing all of the primitives in the device and the legal connections between them, is then created based on the device specified on the command line. (As of now only the SLG46620V is supported; the SLG46621V can be added fairly easily but the SLG46140V has a slightly different microarchitecture which will require a bit more work to support.)</p>
<p> After the graphs are generated, each node in the netlist graph is assigned a numeric label identifying the type of cell and each node in the device graph is assigned a list of legal labels: for example, an I/O buffer site is legal for an input buffer, output buffer, or bidirectional buffer.</p>
<p> Rather than wasting time writing a synthesizer, I decided to write a GreenPak technology
library for Clifford Wolf's excellent open source synthesis tool, <a
href="http://www.clifford.at/yosys/">Yosys</a>, and then make a place-and-route
tool to turn that into a final netlist. The post-PAR netlist can then be loaded into
GreenPak Designer in order to program the device.</p>
<p> The first step of the process is to run the "synth_greenpak4" Yosys flow on the Verilog
source. This runs a generic RTL synthesis pass, then some coarse-grained extraction
passes to infer shift register and counter cells from behavioral logic, and finally maps
the remaining logic to LUT/FF cells and outputs a JSON-formatted netlist.</p>
<p> Once the design has been synthesized, my tool (named, surprisingly, gp4par) is then
launched on the netlist. It begins by parsing the JSON and constructing a directed graph
of cell objects in memory. A second graph, containing all of the primitives in the
device and the legal connections between them, is then created based on the device
specified on the command line. (As of now only the SLG46620V is supported; the SLG46621V
can be added fairly easily but the SLG46140V has a slightly different microarchitecture
which will require a bit more work to support.)</p>
<p> After the graphs are generated, each node in the netlist graph is assigned a numeric
label identifying the type of cell and each node in the device graph is assigned a list
of legal labels: for example, an I/O buffer site is legal for an input buffer, output
buffer, or bidirectional buffer.</p>
<table>
<tbody>
<tr>
<td>
<a href="https://2.bp.blogspot.com/-kIekczO693g/Vy7dBqYifXI/AAAAAAAAAxc/hMNJBs5bedIQOrBzzkhq4gbmhR-n58EQwCLcB/s1600/graph-labels.png" imageanchor="1"><img height="141" src="https://2.bp.blogspot.com/-kIekczO693g/Vy7dBqYifXI/AAAAAAAAAxc/hMNJBs5bedIQOrBzzkhq4gbmhR-n58EQwCLcB/s400/graph-labels.png" width="400" /></a>
</td>
</tr>
<tr>
<td>Example labeling for a subset of the netlist and device graphs</td>
</tr>
<tr>
<td>
<a href="https://2.bp.blogspot.com/-kIekczO693g/Vy7dBqYifXI/AAAAAAAAAxc/hMNJBs5bedIQOrBzzkhq4gbmhR-n58EQwCLcB/s1600/graph-labels.png"
imageanchor="1"><img height="141"
src="https://2.bp.blogspot.com/-kIekczO693g/Vy7dBqYifXI/AAAAAAAAAxc/hMNJBs5bedIQOrBzzkhq4gbmhR-n58EQwCLcB/s400/graph-labels.png"
width="400"/></a>
</td>
</tr>
<tr>
<td>Example labeling for a subset of the netlist and device graphs</td>
</tr>
</tbody>
</table>
<p> The labeled nodes now need to be placed. The initial placement uses a simple greedy algorithm to create a valid (although not necessarily optimal or even routable) placement:</p>
<p> The labeled nodes now need to be placed. The initial placement uses a simple greedy
algorithm to create a valid (although not necessarily optimal or even routable)
placement:</p>
<ol>
<li>Loop over the cells in the netlist. If any cell has a LOC constraint, which locks the cell to a specific physical site, attempt to assign the node to the specified site. If the specified node is the wrong type, doesn't exist, or is already used by another constrained node, the constraint is invalid so fail with an error.</li>
<li>Loop over all of the unconstrained cells in the netlist and assign them to the first unused site with the right label. If none are available, the design is too big for the device so fail with an error.</li>
<li>Loop over the cells in the netlist. If any cell has a LOC constraint, which locks
the cell to a specific physical site, attempt to assign the node to the specified
site. If the specified node is the wrong type, doesn't exist, or is already used by
another constrained node, the constraint is invalid so fail with an error.
</li>
<li>Loop over all of the unconstrained cells in the netlist and assign them to the first
unused site with the right label. If none are available, the design is too big for
the device so fail with an error.
</li>
</ol>
<p> Once the design is placed, the placement optimizer then loops over the design and attempts to improve it. A simulated annealing algorithm is used, where changes to the design are accepted unconditionally if they make the placement better, and with a random, gradually decreasing probability if they make it worse. The optimizer terminates when the design receives a perfect score (indicating an optimal placement) or if it stops making progress for several iterations. Each iteration does the following:</p>
<p> Once the design is placed, the placement optimizer then loops over the design and
attempts to improve it. A simulated annealing algorithm is used, where changes to the
design are accepted unconditionally if they make the placement better, and with a
random, gradually decreasing probability if they make it worse. The optimizer terminates
when the design receives a perfect score (indicating an optimal placement) or if it
stops making progress for several iterations. Each iteration does the following:</p>
<ol>
<li>Compute a score for the current design based on the number of unroutable nets, the amount of routing congestion (number of nets crossing between halves of the device), and static timing analysis (not yet implemented, always zero).</li>
<li>Make a list of nodes that contributed to this score in some way (having some attached nets unroutable, crossing to the other half of the device, or failing timing).</li>
<li>Remove nodes from the list that are LOC'd to a specific location since we're not allowed to move them.</li>
<li>Remove nodes from the list that have only one legal placement in the device (for example, oscillator hard IP) since there's nowhere else for them to go.</li>
<li>Compute a score for the current design based on the number of unroutable nets, the
amount of routing congestion (number of nets crossing between halves of the device),
and static timing analysis (not yet implemented, always zero).
</li>
<li>Make a list of nodes that contributed to this score in some way (having some
attached nets unroutable, crossing to the other half of the device, or failing
timing).
</li>
<li>Remove nodes from the list that are LOC'd to a specific location since we're not
allowed to move them.
</li>
<li>Remove nodes from the list that have only one legal placement in the device (for
example, oscillator hard IP) since there's nowhere else for them to go.
</li>
<li>Pick a node from the remainder of the list at random. Call this our pivot.</li>
<li>Find a list of candidate placements for the pivot: </li>
<li>Find a list of candidate placements for the pivot:</li>
<ol>
<li>Consider all routable placements in the other half of the device.</li>
<li>If none were found, consider all routable placements anywhere in the device.</li>
<li>If none were found, consider all placements anywhere in the device even if they're not routable.</li>
<li>If none were found, consider all routable placements anywhere in the device.
</li>
<li>If none were found, consider all placements anywhere in the device even if
they're not routable.
</li>
</ol>
<li>Pick one of the candidates at random and move the pivot to that location. If another cell in the netlist is already there, put it in the vacant site left by the pivot.</li>
<li>Re-compute the score for the design. If it's better, accept this change and start the next iteration.</li>
<li>If the score is worse, accept it with a random probability which decreases as the iteration number goes up. If the change is not accepted, restore the previous placement.</li>
<li>Pick one of the candidates at random and move the pivot to that location. If another
cell in the netlist is already there, put it in the vacant site left by the pivot.
</li>
<li>Re-compute the score for the design. If it's better, accept this change and start
the next iteration.
</li>
<li>If the score is worse, accept it with a random probability which decreases as the
iteration number goes up. If the change is not accepted, restore the previous
placement.
</li>
</ol>
<p> After optimization, the design is checked for routability. If any edges in the netlist graph don't correspond to edges in the device graph, the user probably asked for something impossible (for example, trying to hook a flipflop's output to a comparator's reference voltage input) so fail with an error.</p>
<p> The design is then routed. This is quite simple due to the crossbar structure of the device. For each edge in the netlist:</p>
<p> After optimization, the design is checked for routability. If any edges in the netlist
graph don't correspond to edges in the device graph, the user probably asked for
something impossible (for example, trying to hook a flipflop's output to a comparator's
reference voltage input) so fail with an error.</p>
<p> The design is then routed. This is quite simple due to the crossbar structure of the
device. For each edge in the netlist:</p>
<ol>
<li>If dedicated (non-fabric) routing is used for this path, configure the destination's input mux appropriately and stop.</li>
<li>If the source and destination are in the same half of the device, configure the destination's input mux appropriately and stop.</li>
<li>A cross-connection must be used. Check if we already used one to bring the source signal to the other half of the device. If found, configure the destination to route from that cross-connection and stop.</li>
<li>Check if we have any cross-connections left going in this direction. If they're all used, the design is unroutable due to congestion so fail with an error.</li>
<li>Pick the next unused cross-connection and configure it to route from the source. Configure the destination to route from the cross-connection and stop.</li>
<li>If dedicated (non-fabric) routing is used for this path, configure the destination's
input mux appropriately and stop.
</li>
<li>If the source and destination are in the same half of the device, configure the
destination's input mux appropriately and stop.
</li>
<li>A cross-connection must be used. Check if we already used one to bring the source
signal to the other half of the device. If found, configure the destination to route
from that cross-connection and stop.
</li>
<li>Check if we have any cross-connections left going in this direction. If they're all
used, the design is unroutable due to congestion so fail with an error.
</li>
<li>Pick the next unused cross-connection and configure it to route from the source.
Configure the destination to route from the cross-connection and stop.
</li>
</ol>
<p> Once routing is finished, run a series of post-PAR design rule checks. These currently include the following:</p>
<p> Once routing is finished, run a series of post-PAR design rule checks. These currently
include the following:</p>
<ul>
<li>If any node has no loads, generate a warning</li>
<li>If an I/O buffer is connected to analog hard IP, fail with an error if it's not configured in analog mode.</li>
<li>Some signals (such as comparator inputs and oscillator power-down controls) are generated by a shared mux and fed to many loads. If different loads require conflicting settings for the shared mux, fail with an error.</li>
<li>If an I/O buffer is connected to analog hard IP, fail with an error if it's not
configured in analog mode.
</li>
<li>Some signals (such as comparator inputs and oscillator power-down controls) are
generated by a shared mux and fed to many loads. If different loads require
conflicting settings for the shared mux, fail with an error.
</li>
</ul>
<p> If DRC passes with no errors, configure all of the individual cells in the netlist based on the HDL parameters. Fail with an error if an invalid configuration was requested.</p>
<p> Finally, generate the bitstream from all of the per-cell configuration and write it to a file.</p>
<p> If DRC passes with no errors, configure all of the individual cells in the netlist based
on the HDL parameters. Fail with an error if an invalid configuration was requested.</p>
<p> Finally, generate the bitstream from all of the per-cell configuration and write it to a
file.</p>
<h2> Great, let's get started!</h2>
<p> If you don't already have one, you'll need to buy a <a href="http://www.silego.com/buy/index.php?main_page=product_info&amp;products_id=388">GreenPak4 development kit</a>. The kit includes samples of the SLG46620V (among other devices) and a programmer/emulation board. While you're waiting for it to arrive, install <a href="http://www.silego.com/softdoc/software.html">GreenPak Designer</a>.</p>
<p> Download and install Yosys. Although Clifford is pretty good at merging my pull requests, only <a href="https://github.com/azonenberg/yosys/">my fork on Github</a> is guaranteed to have the most up-to-date support for GreenPak devices so don't be surprised if you can't use a bleeding-edge feature with mainline Yosys.</p>
<p> Download and install gp4par. You can get it from <a href="https://github.com/azonenberg/openfpga/">the Github repository</a>.</p>
<p> Write your HDL, compile with Yosys, P&amp;R with gp4par, and import the bitstream into GreenPak Designer to program the target device. The most current gp4par manual is included in LaTeX source form in the source tree and is automatically built as part of the compile process. If you're just browsing, there's a <a href="http://thanatos.virtual.antikernel.net/unlisted/gp4-hdl.pdf">relatively recent PDF version</a> on my web server.</p>
<p> If you'd like to see the Verilog that produced the nightmare of a schematic I showed above, <a href="https://github.com/azonenberg/openfpga/blob/master/tests/greenpak4/Blinky/Blinky.v">here it is</a>.</p>
<p> Be advised that this project is still very much a work in progress and there are still a number of SLG46620V features I don't support (see the manual for exact details).</p>
<p> If you don't already have one, you'll need to buy a <a
href="http://www.silego.com/buy/index.php?main_page=product_info&amp;products_id=388">GreenPak4
development kit</a>. The kit includes samples of the SLG46620V (among other devices) and
a programmer/emulation board. While you're waiting for it to arrive, install <a
href="http://www.silego.com/softdoc/software.html">GreenPak Designer</a>.</p>
<p> Download and install Yosys. Although Clifford is pretty good at merging my pull
requests, only <a href="https://github.com/azonenberg/yosys/">my fork on Github</a> is
guaranteed to have the most up-to-date support for GreenPak devices so don't be
surprised if you can't use a bleeding-edge feature with mainline Yosys.</p>
<p> Download and install gp4par. You can get it from <a
href="https://github.com/azonenberg/openfpga/">the Github repository</a>.</p>
<p> Write your HDL, compile with Yosys, P&amp;R with gp4par, and import the bitstream into
GreenPak Designer to program the target device. The most current gp4par manual is
included in LaTeX source form in the source tree and is automatically built as part of
the compile process. If you're just browsing, there's a <a
href="http://thanatos.virtual.antikernel.net/unlisted/gp4-hdl.pdf">relatively
recent PDF version</a> on my web server.</p>
<p> If you'd like to see the Verilog that produced the nightmare of a schematic I showed
above, <a
href="https://github.com/azonenberg/openfpga/blob/master/tests/greenpak4/Blinky/Blinky.v">here
it is</a>.</p>
<p> Be advised that this project is still very much a work in progress and there are still a
number of SLG46620V features I don't support (see the manual for exact details).</p>
<h2> I love it / it segfaulted / there's a problem in the manual!</h2>
<p> Hop in our IRC channel (##openfpga on Freenode) and let me know. Feedback is great, pull requests are even better,</p>
<p> Hop in our IRC channel (##openfpga on Freenode) and let me know. Feedback is great, pull
requests are even better,</p>
<h2> You're competing with Silego's IDE. Have they found out and sued you yet?</h2>
<p> Nope. They're fully aware of what I'm doing and are rolling out the red carpet for me. They love the idea of a HDL flow as an alternative to schematic entry and are pretty amazed at how fast it's coming together.</p>
<p> After I reported a few bugs in their datasheets they decided to skip the middleman and give me direct access to the engineer who writes their documentation so that I can get faster responses. The last time I found a problem (two different parts of the datasheet contradicted each other) an updated datasheet was in my inbox and on their website by the next day. I only wish Xilinx gave me that kind of treatment!</p>
<p> They've even <a href="https://twitter.com/SilegoTech/status/717018987771469824">offered me free hardware</a> to help me add support for their latest product family, although I plan to get GreenPak4 support to a more stable state before taking them up on the offer.</p>
<p> Nope. They're fully aware of what I'm doing and are rolling out the red carpet for me.
They love the idea of a HDL flow as an alternative to schematic entry and are pretty
amazed at how fast it's coming together.</p>
<p> After I reported a few bugs in their datasheets they decided to skip the middleman and
give me direct access to the engineer who writes their documentation so that I can get
faster responses. The last time I found a problem (two different parts of the datasheet
contradicted each other) an updated datasheet was in my inbox and on their website by
the next day. I only wish Xilinx gave me that kind of treatment!</p>
<p> They've even <a href="https://twitter.com/SilegoTech/status/717018987771469824">offered
me free hardware</a> to help me add support for their latest product family, although I
plan to get GreenPak4 support to a more stable state before taking them up on the offer.
</p>
<h2> So what's next?</h2>
<p> Better testing, for starters. I have to verify functionality by hand with a DMM and oscilloscope, which is time consuming.</p>
<p> My contact at Silego says they're going to be giving me documentation on the SRAM emulation interface soon, so I'm going to make a hardware-in-loop test platform that connects to my desktop and the Silego ZIF socket, and lets me load new bitstreams via a scriptable interface. It'll have FPGA-based digital I/O as well as an ADC and DAC on every device pin, plus an adjustable voltage regulator for power, so I can feed in arbitrary mixed-signal test waveforms and write PC-based unit tests to verify correct behavior.</p>
<p> Other than that, I want to finish support for the SLG46620V in the next month or two. The SLG46621V will be an easy addition since only one pin and the relevant configuration bits have changed from the 46620 (I suspect they're the same die, just bonded out differently).</p>
<p> Once that's done I'll have to do some more extensive work to add the SLG46140V since the architecture is a bit different (a lot of the combinatorial logic is merged into multi-function blocks). Luckily, the 46140 has a lot in common architecturally with the GreenPak5 family, so once that's done GreenPak5 will probably be a lot easier to add support for.</p>
<p> My thanks go out to Clifford Wolf, whitequark, the IRC users in ##openfpga, and everyone at Silego I've worked with to help make this possible. I hope that one day this project will become mature enough that Silego will ship it as an officially supported extension to GreenPak Designer, making history by becoming the first modern programmable logic vendor to ship a fully open source synthesis and P&amp;R suite. </p>
<p> Better testing, for starters. I have to verify functionality by hand with a DMM and
oscilloscope, which is time consuming.</p>
<p> My contact at Silego says they're going to be giving me documentation on the SRAM
emulation interface soon, so I'm going to make a hardware-in-loop test platform that
connects to my desktop and the Silego ZIF socket, and lets me load new bitstreams via a
scriptable interface. It'll have FPGA-based digital I/O as well as an ADC and DAC on
every device pin, plus an adjustable voltage regulator for power, so I can feed in
arbitrary mixed-signal test waveforms and write PC-based unit tests to verify correct
behavior.</p>
<p> Other than that, I want to finish support for the SLG46620V in the next month or two.
The SLG46621V will be an easy addition since only one pin and the relevant configuration
bits have changed from the 46620 (I suspect they're the same die, just bonded out
differently).</p>
<p> Once that's done I'll have to do some more extensive work to add the SLG46140V since the
architecture is a bit different (a lot of the combinatorial logic is merged into
multi-function blocks). Luckily, the 46140 has a lot in common architecturally with the
GreenPak5 family, so once that's done GreenPak5 will probably be a lot easier to add
support for.</p>
<p> My thanks go out to Clifford Wolf, whitequark, the IRC users in ##openfpga, and everyone
at Silego I've worked with to help make this possible. I hope that one day this project
will become mature enough that Silego will ship it as an officially supported extension
to GreenPak Designer, making history by becoming the first modern programmable logic
vendor to ship a fully open source synthesis and P&amp;R suite. </p>
</div>
</div>

2947
app/src/main/assets/readability/test/test-pages/blogger/source.html
View File

File diff suppressed because it is too large Load Diff

33
app/src/main/assets/readability/test/test-pages/breitbart/expected.html
View File

@ -2,7 +2,10 @@
<div>
<figure>
<div>
<p><img itemprop="image" src="http://media.breitbart.com/media/2016/11/GettyImages-621866810-640x480.jpg" alt="Supporters of Republican presidential nominee Donald Trump cheer during election night at the New York Hilton Midtown in New York on November 9, 2016. / AFP / JIM WATSON (Photo credit should read JIM WATSON/AFP/Getty Images)" width="640" height="480" /></p>
<p><img itemprop="image"
src="http://media.breitbart.com/media/2016/11/GettyImages-621866810-640x480.jpg"
alt="Supporters of Republican presidential nominee Donald Trump cheer during election night at the New York Hilton Midtown in New York on November 9, 2016. / AFP / JIM WATSON (Photo credit should read JIM WATSON/AFP/Getty Images)"
width="640" height="480"/></p>
<p>JIM WATSON/AFP/Getty Images</p>
</div>
</figure>
@ -13,15 +16,29 @@
<div id="EmailOptin">
<p><span>SIGN UP</span> FOR OUR NEWSLETTER</p>
</div>
<h2><span>Snopes fact checker and staff writer David Emery posted to Twitter asking if there were “any un-angry Trump supporters?”</span></h2>
<p><span>Emery, a writer for partisan “fact-checking” website Snopes.com which soon will be in charge of labelling </span><a href="http://www.breitbart.com/tech/2016/12/15/facebook-introduce-warning-labels-stories-deemed-fake-news/"><span>“fake news”</span></a><span> alongside ABC News and Politifact, retweeted an article by Vulture magazine relating to the </span><a href="http://www.breitbart.com/big-hollywood/2016/11/19/boycotthamilton-trends-hamilton-cast-members-harass-mike-pence/"><span>protests</span></a><span> of the <em>Hamilton</em> musical following the decision by the cast of the show to make a </span><a href="http://www.breitbart.com/big-hollywood/2016/11/19/tolerance-hamilton-cast-lectures-mike-pence-broadway-stage/"><span>public announcement</span></a><span> to Vice-president elect Mike Pence while he watched the performance with his family.</span></p>
<h2><span>Snopes fact checker and staff writer David Emery posted to Twitter asking if there were “any un-angry Trump supporters?”</span>
</h2>
<p><span>Emery, a writer for partisan “fact-checking” website Snopes.com which soon will be in charge of labelling </span><a
href="http://www.breitbart.com/tech/2016/12/15/facebook-introduce-warning-labels-stories-deemed-fake-news/"><span>“fake news”</span></a><span> alongside ABC News and Politifact, retweeted an article by Vulture magazine relating to the </span><a
href="http://www.breitbart.com/big-hollywood/2016/11/19/boycotthamilton-trends-hamilton-cast-members-harass-mike-pence/"><span>protests</span></a><span> of the <em>Hamilton</em> musical following the decision by the cast of the show to make a </span><a
href="http://www.breitbart.com/big-hollywood/2016/11/19/tolerance-hamilton-cast-lectures-mike-pence-broadway-stage/"><span>public announcement</span></a><span> to Vice-president elect Mike Pence while he watched the performance with his family.</span>
</p>
<div id="EmailOptinM">
<p><span>SIGN UP</span> FOR OUR NEWSLETTER</p>
</div>
<p><span>The tweet from Vulture magazine reads, “</span><a href="https://twitter.com/hashtag/Hamilton?src=hash" target="_blank" rel="noopener"><span>#Hamilton</span></a><span> Chicago show interrupted by angry Trump supporter.” Emery retweeted the story, saying, “Are there un-angry Trump supporters?”</span></p>
<p><span>This isn’t the first time the Snopes.com writer has expressed anti-Trump sentiment on his Twitter page. In another tweet in which Emery links to an article that falsely attributes a quote to President-elect Trump, Emery states, “Incredibly, some people actually think they have to put words in Trump’s mouth to make him look bad.”</span></p>
<p><span>Emery also retweeted an article by <em>New York</em> magazine that claimed President-elect Trump relied on lies to win during his campaign and that we now lived in a “post-truth” society. “Before long we’ll all have forgotten what it was like to live in the same universe; or maybe we already have,” Emery tweeted.</span></p>
<p><span>Facebook believe that Emery, along with other Snopes writers, ABC News, and </span><a href="http://www.breitbart.com/tech/2016/12/16/flashback-weekly-standard-data-shows-politifact-has-it-out-for-republicans/"><span>Politifact</span></a><span> are impartial enough to label and silence what they believe to be “fake news” on social media. </span></p>
<p><i><span>Lucas Nolan is a reporter for Breitbart Tech covering issues of free speech and online censorship. Follow him on Twitter </span></i><a href="http://twitter.com/lucasnolan_" target="_blank" rel="noopener"><i><span>@LucasNolan_</span></i></a><i><span> or email him at </span></i><a href="http://www.breitbart.com/wp-admin/blank"><i><span>lnolan@breitbart.com</span></i></a></p>
<p><span>The tweet from Vulture magazine reads, “</span><a
href="https://twitter.com/hashtag/Hamilton?src=hash" target="_blank" rel="noopener"><span>#Hamilton</span></a><span> Chicago show interrupted by angry Trump supporter.” Emery retweeted the story, saying, “Are there un-angry Trump supporters?”</span>
</p>
<p><span>This isn’t the first time the Snopes.com writer has expressed anti-Trump sentiment on his Twitter page. In another tweet in which Emery links to an article that falsely attributes a quote to President-elect Trump, Emery states, “Incredibly, some people actually think they have to put words in Trump’s mouth to make him look bad.”</span>
</p>
<p><span>Emery also retweeted an article by <em>New York</em> magazine that claimed President-elect Trump relied on lies to win during his campaign and that we now lived in a “post-truth” society. “Before long we’ll all have forgotten what it was like to live in the same universe; or maybe we already have,” Emery tweeted.</span>
</p>
<p><span>Facebook believe that Emery, along with other Snopes writers, ABC News, and </span><a
href="http://www.breitbart.com/tech/2016/12/16/flashback-weekly-standard-data-shows-politifact-has-it-out-for-republicans/"><span>Politifact</span></a><span> are impartial enough to label and silence what they believe to be “fake news” on social media. </span>
</p>
<p><i><span>Lucas Nolan is a reporter for Breitbart Tech covering issues of free speech and online censorship. Follow him on Twitter </span></i><a
href="http://twitter.com/lucasnolan_" target="_blank" rel="noopener"><i><span>@LucasNolan_</span></i></a><i><span> or email him at </span></i><a
href="http://www.breitbart.com/wp-admin/blank"><i><span>lnolan@breitbart.com</span></i></a>
</p>
</div>
</div>

4366
app/src/main/assets/readability/test/test-pages/breitbart/source.html
View File

File diff suppressed because one or more lines are too long

93
app/src/main/assets/readability/test/test-pages/bug-1255978/expected.html
View File

@ -1,64 +1,115 @@
<div id="readability-page-1" class="page">
<div itemprop="articleBody" id="gigya-share-btns-2_gig_containerParent">
<p>Most people go to hotels for the pleasure of sleeping in a giant bed with clean white sheets and waking up to fresh towels in the morning.</p>
<p>But those towels and sheets might not be as clean as they look, according to the hotel bosses that responded to an online thread about the things hotel owners don’t want you to know.</p>
<p>Zeev Sharon and Michael Forrest Jones both run hotel start-ups in the US. Forrest Jones runs the start-up Beechmont Hotels Corporation, a hotel operating company that consults with hotel owners on how they can improve their business. Sharon is the CEO of Hotelied, a start-up that allows people to sign up for discounts at luxury hotels.</p>
<p>Most people go to hotels for the pleasure of sleeping in a giant bed with clean white
sheets and waking up to fresh towels in the morning.</p>
<p>But those towels and sheets might not be as clean as they look, according to the hotel
bosses that responded to an online thread about the things hotel owners don’t want you
to know.</p>
<p>Zeev Sharon and Michael Forrest Jones both run hotel start-ups in the US. Forrest Jones
runs the start-up Beechmont Hotels Corporation, a hotel operating company that consults
with hotel owners on how they can improve their business. Sharon is the CEO of Hotelied,
a start-up that allows people to sign up for discounts at luxury hotels.</p>
<p>But even luxury hotels aren’t always cleaned as often as they should be.</p>
<p>Here are some of the secrets that the receptionist will never tell you when you check in, according to answers posted on <a href="https://www.quora.com/What-are-the-things-we-dont-know-about-hotel-rooms" target="_blank">Quora</a>.</p>
<p>Here are some of the secrets that the receptionist will never tell you when you check in,
according to answers posted on <a
href="https://www.quora.com/What-are-the-things-we-dont-know-about-hotel-rooms"
target="_blank">Quora</a>.</p>
<div>
<div>
<p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2014/03/18/10/bandb2.jpg" alt="bandb2.jpg" title="bandb2.jpg" width="564" height="423" /></p>
<p>
<img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2014/03/18/10/bandb2.jpg"
alt="bandb2.jpg" title="bandb2.jpg" width="564" height="423"/></p>
</div>
<p>Even posh hotels might not wash a blanket in between stays </p>
</div>
<p>1. Take any blankets or duvets off the bed</p>
<p>Forrest Jones said that anything that comes into contact with any of the previous guest’s skin should be taken out and washed every time the room is made, but that even the fanciest hotels don’t always do so. "Hotels are getting away from comforters. Blankets are here to stay, however. But some hotels are still hesitant about washing them every day if they think they can get out of it," he said.</p>
<p>Forrest Jones said that anything that comes into contact with any of the previous guest’s
skin should be taken out and washed every time the room is made, but that even the
fanciest hotels don’t always do so. "Hotels are getting away from comforters. Blankets
are here to stay, however. But some hotels are still hesitant about washing them every
day if they think they can get out of it," he said.</p>
<div>
<p>Video shows bed bug infestation at New York hotel</p>
</div>
<div>
<div>
<p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2015/05/26/11/hotel-door-getty.jpg" alt="hotel-door-getty.jpg" title="hotel-door-getty.jpg" width="564" height="423" /></p>
<p>
<img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2015/05/26/11/hotel-door-getty.jpg"
alt="hotel-door-getty.jpg" title="hotel-door-getty.jpg" width="564"
height="423"/></p>
</div>
<p>Forrest Jones advised stuffing the peep hole with a strip of rolled up notepaper when not in use. </p>
<p>Forrest Jones advised stuffing the peep hole with a strip of rolled up notepaper when
not in use. </p>
</div>
<p>2. Check the peep hole has not been tampered with</p>
<p>This is not common, but can happen, Forrest Jones said. He advised stuffing the peep hole with a strip of rolled up notepaper when not in use. When someone knocks on the door, the paper can be removed to check who is there. If no one is visible, he recommends calling the front desk immediately. “I look forward to the day when I can tell you to choose only hotels where every employee who has access to guestroom keys is subjected to a complete public records background check, prior to hire, and every year or two thereafter. But for now, I can't,” he said.</p>
<p>This is not common, but can happen, Forrest Jones said. He advised stuffing the peep hole
with a strip of rolled up notepaper when not in use. When someone knocks on the door,
the paper can be removed to check who is there. If no one is visible, he recommends
calling the front desk immediately. “I look forward to the day when I can tell you to
choose only hotels where every employee who has access to guestroom keys is subjected to
a complete public records background check, prior to hire, and every year or two
thereafter. But for now, I can't,” he said.</p>
<div>
<div>
<p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2013/07/31/15/luggage-3.jpg" alt="luggage-3.jpg" title="luggage-3.jpg" width="564" height="423" /></p>
<p>
<img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2013/07/31/15/luggage-3.jpg"
alt="luggage-3.jpg" title="luggage-3.jpg" width="564" height="423"/></p>
</div>
<p>Put luggage on the floor </p>
</div>
<p>3. Don’t use a wooden luggage rack</p>
<p>Bedbugs love wood. Even though a wooden luggage rack might look nicer and more expensive than a metal one, it’s a breeding ground for bugs. Forrest Jones says guests should put the items they plan to take from bags on other pieces of furniture and leave the bag on the floor.</p>
<p>Bedbugs love wood. Even though a wooden luggage rack might look nicer and more expensive
than a metal one, it’s a breeding ground for bugs. Forrest Jones says guests should put
the items they plan to take from bags on other pieces of furniture and leave the bag on
the floor.</p>
<div>
<div>
<p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2015/04/13/11/Lifestyle-hotels.jpg" alt="Lifestyle-hotels.jpg" title="Lifestyle-hotels.jpg" width="564" height="423" /></p>
<p>
<img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2015/04/13/11/Lifestyle-hotels.jpg"
alt="Lifestyle-hotels.jpg" title="Lifestyle-hotels.jpg" width="564"
height="423"/></p>
</div>
<p>The old rule of thumb is that for every 00 invested in a room, the hotel should charge in average daily rate </p>
<p>The old rule of thumb is that for every 00 invested in a room, the hotel should
charge in average daily rate </p>
</div>
<p>4. Hotel rooms are priced according to how expensive they were to build</p>
<p>Zeev Sharon said that the old rule of thumb is that for every $1000 invested in a room, the hotel should charge $1 in average daily rate. So a room that cost $300,000 to build, should sell on average for $300/night.</p>
<p>Zeev Sharon said that the old rule of thumb is that for every $1000 invested in a room,
the hotel should charge $1 in average daily rate. So a room that cost $300,000 to build,
should sell on average for $300/night.</p>
<h3>5. Beware the wall-mounted hairdryer</h3>
<p>It contains the most germs of anything in the room. Other studies have said the TV remote and bedside lamp switches are the most unhygienic. “Perhaps because it's something that's easy for the housekeepers to forget to check or to squirt down with disinfectant,” Forrest Jones said.</p>
<p>It contains the most germs of anything in the room. Other studies have said the TV remote
and bedside lamp switches are the most unhygienic. “Perhaps because it's something
that's easy for the housekeepers to forget to check or to squirt down with
disinfectant,” Forrest Jones said.</p>
<h3>6. Mini bars almost always lose money</h3>
<p>Despite the snacks in the minibar seeming like the most overpriced food you have ever seen, hotel owners are still struggling to make a profit from those snacks. "Minibars almost always lose money, even when they charge $10 for a Diet Coke,” Sharon said.</p>
<p>Despite the snacks in the minibar seeming like the most overpriced food you have ever
seen, hotel owners are still struggling to make a profit from those snacks. "Minibars
almost always lose money, even when they charge $10 for a Diet Coke,” Sharon said.</p>
<div>
<div>
<p><img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2014/03/13/16/agenda7.jpg" alt="agenda7.jpg" title="agenda7.jpg" width="564" height="423" /></p>
<p>
<img src="https://static.independent.co.uk/s3fs-public/styles/story_medium/public/thumbnails/image/2014/03/13/16/agenda7.jpg"
alt="agenda7.jpg" title="agenda7.jpg" width="564" height="423"/></p>
</div>
<p>Towels should always be cleaned between stays </p>
</div>
<p>7. Always made sure the hand towels are clean when you arrive</p>
<p>Forrest Jones made a discovery when he was helping out with the housekeepers. “You know where you almost always find a hand towel in any recently-vacated hotel room that was occupied by a guy? On the floor, next to the bed, about halfway down, maybe a little toward the foot of the bed. Same spot in the floor, next to almost every bed occupied by a man, in every room. I'll leave the rest to your imagination,” he said.</p>
<meta itemprop="datePublished" content="2016-05-08T10:11:51+01:00" />
<p>Forrest Jones made a discovery when he was helping out with the housekeepers. “You know
where you almost always find a hand towel in any recently-vacated hotel room that was
occupied by a guy? On the floor, next to the bed, about halfway down, maybe a little
toward the foot of the bed. Same spot in the floor, next to almost every bed occupied by
a man, in every room. I'll leave the rest to your imagination,” he said.</p>
<meta itemprop="datePublished" content="2016-05-08T10:11:51+01:00"/>
<ul>
<li> More about: </li>
<li> More about:</li>
<li><a itemprop="keywords" href="http://fakehost/topic/Hotels">Hotels</a></li>
<li><a itemprop="keywords" href="http://fakehost/topic/Hygiene">Hygiene</a></li>
</ul>
<p><a href="http://fakehost/syndication/reuse-permision-form?url=http://www.independent.co.uk/news/business/news/seven-secrets-that-hotel-owners-dont-want-you-to-know-10506160.html" target="_blank"><img src="http://fakehost/sites/all/themes/ines_themes/independent_theme/img/reuse.png" width="25" />Reuse content</a>
<p>
<a href="http://fakehost/syndication/reuse-permision-form?url=http://www.independent.co.uk/news/business/news/seven-secrets-that-hotel-owners-dont-want-you-to-know-10506160.html"
target="_blank"><img
src="http://fakehost/sites/all/themes/ines_themes/independent_theme/img/reuse.png"
width="25"/>Reuse content</a>
</p>
</div>
</div>

6462
app/src/main/assets/readability/test/test-pages/bug-1255978/source.html
View File

File diff suppressed because it is too large Load Diff

50
app/src/main/assets/readability/test/test-pages/buzzfeed-1/expected.html
View File

@ -1,39 +1,63 @@
<div id="readability-page-1" class="page">
<div id="buzz_sub_buzz">
<div id="superlist_3758406_5547137" rel:buzz_num="1">
<h2>The mother of a woman who took suspected diet pills bought online has described how her daughter was “literally burning up from within” moments before her death.</h2>
<p> <span>West Merica Police</span></p>
<h2>The mother of a woman who took suspected diet pills bought online has described how
her daughter was “literally burning up from within” moments before her death.</h2>
<p><span>West Merica Police</span></p>
</div>
<div id="superlist_3758406_5547213" rel:buzz_num="2">
<p>Eloise Parry, 21, was taken to Royal Shrewsbury hospital on 12 April after taking a lethal dose of highly toxic “slimming tablets”. </p>
<p>“The drug was in her system, there was no anti-dote, two tablets was a lethal dose – and she had taken eight,” her mother, Fiona, <a href="https://www.westmercia.police.uk/article/9501/A-tribute-to-Eloise-Aimee-Parry-written-by-her-mother-Fiona-Parry">said in a statement</a> yesterday.</p>
<p>“As Eloise deteriorated, the staff in A&amp;E did all they could to stabilise her. As the drug kicked in and started to make her metabolism soar, they attempted to cool her down, but they were fighting an uphill battle.</p>
<p>Eloise Parry, 21, was taken to Royal Shrewsbury hospital on 12 April after taking a
lethal dose of highly toxic “slimming tablets”. </p>
<p>“The drug was in her system, there was no anti-dote, two tablets was a lethal dose –
and she had taken eight,” her mother, Fiona, <a
href="https://www.westmercia.police.uk/article/9501/A-tribute-to-Eloise-Aimee-Parry-written-by-her-mother-Fiona-Parry">said
in a statement</a> yesterday.</p>
<p>“As Eloise deteriorated, the staff in A&amp;E did all they could to stabilise her. As
the drug kicked in and started to make her metabolism soar, they attempted to cool
her down, but they were fighting an uphill battle.</p>
<p>“She was literally burning up from within.”</p>
<p>She added: “They never stood a chance of saving her. She burned and crashed.”</p>
</div>
<div id="superlist_3758406_5547140" rel:buzz_num="3">
<div>
<div>
<p><img src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608056-15.jpg" rel:bf_image_src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608056-15.jpg" height="412" width="203" /></p>
<p>
<img src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608056-15.jpg"
rel:bf_image_src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608056-15.jpg"
height="412" width="203"/></p>
</div>
<p>Facebook</p>
</div>
<div>
<div>
<p><img src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608057-18.jpg" rel:bf_image_src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608057-18.jpg" height="412" width="412" /></p>
<p>
<img src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608057-18.jpg"
rel:bf_image_src="http://ak-hdl.buzzfed.com/static/2015-04/21/5/enhanced/webdr12/grid-cell-2501-1429608057-18.jpg"
height="412" width="412"/></p>
</div>
<p>Facebook</p>
</div>
</div>
<div id="superlist_3758406_5547284" rel:buzz_num="4">
<p>West Mercia police <a href="https://www.westmercia.police.uk/article/9500/Warning-Issued-As-Shrewsbury-Woman-Dies-After-Taking-Suspected-Diet-Pills">said the tablets were believed to contain dinitrophenol</a>, known as DNP, which is a highly toxic industrial chemical. </p>
<p>“We are undoubtedly concerned over the origin and sale of these pills and are working with partner agencies to establish where they were bought from and how they were advertised,” said chief inspector Jennifer Mattinson from the West Mercia police.</p>
<p>The Food Standards Agency warned people to stay away from slimming products that contained DNP.</p>
<p>“We advise the public not to take any tablets or powders containing DNP, as it is an industrial chemical and not fit for human consumption,” it said in a statement.</p>
<p>West Mercia police <a
href="https://www.westmercia.police.uk/article/9500/Warning-Issued-As-Shrewsbury-Woman-Dies-After-Taking-Suspected-Diet-Pills">said
the tablets were believed to contain dinitrophenol</a>, known as DNP, which is a
highly toxic industrial chemical. </p>
<p>“We are undoubtedly concerned over the origin and sale of these pills and are working
with partner agencies to establish where they were bought from and how they were
advertised,” said chief inspector Jennifer Mattinson from the West Mercia
police.</p>
<p>The Food Standards Agency warned people to stay away from slimming products that
contained DNP.</p>
<p>“We advise the public not to take any tablets or powders containing DNP, as it is an
industrial chemical and not fit for human consumption,” it said in a statement.</p>
</div>
<div id="superlist_3758406_5547219" rel:buzz_num="5">
<h2>Fiona Parry issued a plea for people to stay away from pills containing the chemical.</h2>
<p>“[Eloise] just never really understood how dangerous the tablets that she took were,” she said. “Most of us don’t believe that a slimming tablet could possibly kill us.</p>
<h2>Fiona Parry issued a plea for people to stay away from pills containing the
chemical.</h2>
<p>“[Eloise] just never really understood how dangerous the tablets that she took were,”
she said. “Most of us don’t believe that a slimming tablet could possibly kill
us.</p>
<p>“DNP is not a miracle slimming pill. It is a deadly toxin.”</p>
</div>
</div>

4274
app/src/main/assets/readability/test/test-pages/buzzfeed-1/source.html
View File

File diff suppressed because one or more lines are too long

199
app/src/main/assets/readability/test/test-pages/citylab-1/expected.html
View File

@ -1,69 +1,206 @@
<div id="readability-page-1" class="page">
<article itemscope="itemscope" itemtype="https://schema.org/NewsArticle">
<meta itemprop="datePublished" content="2019-04-30T13:39:00-04:00" />
<meta itemprop="dateModified" content="2019-04-30T13:40:00-04:00" />
<meta itemprop="mainEntityOfPage" content="https://www.citylab.com/design/2019/04/neon-signage-20th-century-history/588400/" />
<meta itemprop="datePublished" content="2019-04-30T13:39:00-04:00"/>
<meta itemprop="dateModified" content="2019-04-30T13:40:00-04:00"/>
<meta itemprop="mainEntityOfPage"
content="https://www.citylab.com/design/2019/04/neon-signage-20th-century-history/588400/"/>
<figure itemprop="image" itemscope="itemscope" itemtype="http://schema.org/ImageObject">
<picture>
<source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/940.jpg?mod=1556645448" media="(min-width: 1024px)" />
<source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/lead_large.jpg?mod=1556645448" media="(min-width: 576px)" />
<source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/940.jpg?mod=1556645448"
media="(min-width: 1024px)"/>
<source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/lead_large.jpg?mod=1556645448"
media="(min-width: 576px)"/>
</picture>
<meta itemprop="height" content="128" />
<meta itemprop="width" content="300" />
<meta itemprop="url" content="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" />
<meta itemprop="height" content="128"/>
<meta itemprop="width" content="300"/>
<meta itemprop="url"
content="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"/>
<picture>
<source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" media="(max-width: 575px)" /><img src="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" alt="" srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" />
<source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"
media="(max-width: 575px)"/>
<img src="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"
alt=""
srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"/>
</picture>
<figcaption>
<span itemprop="caption">The Moulin Rouge cabaret in Paris</span> <span itemprop="creator">Benoit Tessier/Reuters</span>
<span itemprop="caption">The Moulin Rouge cabaret in Paris</span> <span
itemprop="creator">Benoit Tessier/Reuters</span>
</figcaption>
</figure>
<div>
<h2 itemprop="headline"> Why Neon Is the Ultimate Symbol of the 20th Century </h2>
</div>
<h2 itemprop="description"> The once-ubiquitous form of lighting was novel when it first emerged in the early 1900s, though it has since come to represent decline. </h2>
<h2 itemprop="description"> The once-ubiquitous form of lighting was novel when it first
emerged in the early 1900s, though it has since come to represent decline. </h2>
<section id="article-section-1">
<p> In the summer of 1898, the Scottish chemist Sir William Ramsay made a discovery that would eventually give the Moulin Rouge in Paris, the Las Vegas Strip, and New York’s Times Square their perpetual nighttime glow. Using the boiling point of argon as a reference point, Ramsay and his colleague Morris W. Travers isolated three more noble gases and gave them evocative Greek names: neon, krypton, and xenon. In so doing, the scientists bestowed a label of permanent novelty on the most famous of the trio—neon, which translates as “new.” This discovery was the foundation on which the French engineer Georges Claude crafted a new form of illumination over the next decade. He designed glass tubes in which neon gas could be trapped, then electrified, to create a light that glowed reliably for more than 1,000 hours. </p>
<p> In the 2012 book <em>L’être et le Néon</em>, <a href="https://mitpress.mit.edu/books/being-and-neonness-translation-and-content-revised-augmented-and-updated-edition-luis-de-miranda">which has been newly translated into English by Michael Wells</a>, the philosopher Luis de Miranda weaves a history of neon lighting as both artifact and metaphor. <em>Being and Neonness</em>, as the book is called in its English edition, isn’t a typical material history. There are no photographs. Even de Miranda’s own example of a neon deli sign spotted in Paris is re-created typographically, with text in all caps and dashes forming the border of the sign, as one might attempt on Twitter. Fans of Miami Beach’s restored Art Deco hotels and California’s bowling alleys might be disappointed by the lack of glossy historical images. Nonetheless, de Miranda makes a convincing case for neon as a symbol of the grand modern ambitions of the 20th century. </p>
<p> De Miranda beautifully evokes the notion of neon lighting as an icon of the 1900s in his introduction: “When we hear the word <em>neon</em>, an image pops into our heads: a combination of light, colors, symbols, and glass. This image is itself a mood. It carries an atmosphere. It speaks … of the essence of cities, of the poetry of nights, of the 20th century.” When neon lights debuted in Europe, they seemed dazzlingly futuristic. But their husky physicality started becoming obsolete by the 1960s, thanks in part to the widespread use of plastic for fluorescent signs. Neon signs exist today, though they’ve been eclipsed by newer technologies such as digital billboards, and they remain charmingly analog: Signs must be made by hand because there’s no cost-effective way to mass-produce them. </p>
<p> In the 1910s, neon started being used for cosmopolitan flash in Paris at precisely the time and place where the first great modernist works were being created. De Miranda’s recounting of the ingenuity emerging from the French capital a century ago is thrilling to contemplate: the cubist art of Pablo Picasso, the radically deconstructed fashions of Coco Chanel, the stream-of-consciousness poetry of Gertrude Stein, and the genre-defying music of Claude Debussy—all of which heralded a new age of culture for Europe and for the world. </p>
<p> In the summer of 1898, the Scottish chemist Sir William Ramsay made a discovery that
would eventually give the Moulin Rouge in Paris, the Las Vegas Strip, and New York’s
Times Square their perpetual nighttime glow. Using the boiling point of argon as a
reference point, Ramsay and his colleague Morris W. Travers isolated three more
noble gases and gave them evocative Greek names: neon, krypton, and xenon. In so
doing, the scientists bestowed a label of permanent novelty on the most famous of
the trio—neon, which translates as “new.” This discovery was the foundation on which
the French engineer Georges Claude crafted a new form of illumination over the next
decade. He designed glass tubes in which neon gas could be trapped, then
electrified, to create a light that glowed reliably for more than 1,000 hours. </p>
<p> In the 2012 book <em>L’être et le Néon</em>, <a
href="https://mitpress.mit.edu/books/being-and-neonness-translation-and-content-revised-augmented-and-updated-edition-luis-de-miranda">which
has been newly translated into English by Michael Wells</a>, the philosopher Luis de
Miranda weaves a history of neon lighting as both artifact and metaphor. <em>Being
and Neonness</em>, as the book is called in its English edition, isn’t a typical
material history. There are no photographs. Even de Miranda’s own example of a neon
deli sign spotted in Paris is re-created typographically, with text in all caps and
dashes forming the border of the sign, as one might attempt on Twitter. Fans of
Miami Beach’s restored Art Deco hotels and California’s bowling alleys might be
disappointed by the lack of glossy historical images. Nonetheless, de Miranda makes
a convincing case for neon as a symbol of the grand modern ambitions of the 20th
century. </p>
<p> De Miranda beautifully evokes the notion of neon lighting as an icon of the 1900s in
his introduction: “When we hear the word <em>neon</em>, an image pops into our
heads: a combination of light, colors, symbols, and glass. This image is itself a
mood. It carries an atmosphere. It speaks … of the essence of cities, of the poetry
of nights, of the 20th century.” When neon lights debuted in Europe, they seemed
dazzlingly futuristic. But their husky physicality started becoming obsolete by the
1960s, thanks in part to the widespread use of plastic for fluorescent signs. Neon
signs exist today, though they’ve been eclipsed by newer technologies such as
digital billboards, and they remain charmingly analog: Signs must be made by hand
because there’s no cost-effective way to mass-produce them. </p>
<p> In the 1910s, neon started being used for cosmopolitan flash in Paris at precisely
the time and place where the first great modernist works were being created. De
Miranda’s recounting of the ingenuity emerging from the French capital a century ago
is thrilling to contemplate: the cubist art of Pablo Picasso, the radically
deconstructed fashions of Coco Chanel, the stream-of-consciousness poetry of
Gertrude Stein, and the genre-defying music of Claude Debussy—all of which heralded
a new age of culture for Europe and for the world. </p>
</section>
<section id="article-section-2">
<p> Amid this artistic groundswell, Georges Claude premiered his neon lights at the <a href="https://www.mondial-paris.com/en/visiteur/auto">Paris Motor Show</a> in December 1910, captivating visitors with 40-foot-tall tubes affixed to the building’s exterior. The lights shone orange-red because neon, by itself, produces that color. <em>Neon lighting</em> is a catchall term that describes the technology of glass tubing that contains gas or chemicals that glow when electrified. For example, neon fabricators use carbon dioxide to make white, and mercury to make blue. Claude acknowledged at the time that neon didn’t produce the ideal color for a standard light bulb and insisted that it posed no commercial threat to incandescent bulbs. </p>
<p> Of course, the very quality that made neon fixtures a poor choice for interior lighting made them perfect for signs, de Miranda notes. The first of the neon signs was switched on in 1912, advertising a barbershop on Paris’s Boulevard Montmartre, and eventually they were adopted by cinemas and nightclubs. While Claude had a monopoly on neon lighting throughout the 1920s, the leaking of trade secrets and the expiration of a series of patents broke his hold on the rapidly expanding technology. </p>
<p> Amid this artistic groundswell, Georges Claude premiered his neon lights at the <a
href="https://www.mondial-paris.com/en/visiteur/auto">Paris Motor Show</a> in
December 1910, captivating visitors with 40-foot-tall tubes affixed to the
building’s exterior. The lights shone orange-red because neon, by itself, produces
that color. <em>Neon lighting</em> is a catchall term that describes the technology
of glass tubing that contains gas or chemicals that glow when electrified. For
example, neon fabricators use carbon dioxide to make white, and mercury to make
blue. Claude acknowledged at the time that neon didn’t produce the ideal color for a
standard light bulb and insisted that it posed no commercial threat to incandescent
bulbs. </p>
<p> Of course, the very quality that made neon fixtures a poor choice for interior
lighting made them perfect for signs, de Miranda notes. The first of the neon signs
was switched on in 1912, advertising a barbershop on Paris’s Boulevard Montmartre,
and eventually they were adopted by cinemas and nightclubs. While Claude had a
monopoly on neon lighting throughout the 1920s, the leaking of trade secrets and the
expiration of a series of patents broke his hold on the rapidly expanding
technology. </p>
</section>
<section id="article-section-3">
<p> In the following decades, neon’s nonstop glow and vibrant colors turned ordinary buildings and surfaces into 24/7 billboards for businesses, large and small, that wanted to convey a sense of always being open. The first examples of neon in the United States debuted in Los Angeles, where the Packard Motor Car Company commissioned two large blue-and-orange <span>Packard</span> signs that literally stopped traffic because they distracted motorists. The lighting also featured heavily at the Chicago Century of Progress Exposition in 1933 and at the 1939 World’s Fair in New York. At the latter event, a massive neon sign reading <span>Futurama</span> lit the way to a General Motors exhibition that heralded “The World of Tomorrow.” </p>
<p> In the following decades, neon’s nonstop glow and vibrant colors turned ordinary
buildings and surfaces into 24/7 billboards for businesses, large and small, that
wanted to convey a sense of always being open. The first examples of neon in the
United States debuted in Los Angeles, where the Packard Motor Car Company
commissioned two large blue-and-orange <span>Packard</span> signs that literally
stopped traffic because they distracted motorists. The lighting also featured
heavily at the Chicago Century of Progress Exposition in 1933 and at the 1939
World’s Fair in New York. At the latter event, a massive neon sign reading <span>Futurama</span>
lit the way to a General Motors exhibition that heralded “The World of Tomorrow.”
</p>
<figure>
<picture><img alt="" data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg" src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg" /></picture>
<figcaption class="caption"> Workers remove a hammer and sickle from a neon sign that reads “Glory to Communism,” visible on the roof of the Communist-run electricity-board headquarters in Czechoslovakia in 1989. (AP) </figcaption>
<picture><img alt=""
data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg"
src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg"/>
</picture>
<figcaption class="caption"> Workers remove a hammer and sickle from a neon sign
that reads “Glory to Communism,” visible on the roof of the Communist-run
electricity-board headquarters in Czechoslovakia in 1989. (AP)
</figcaption>
</figure>
<p> De Miranda points out that businesses weren’t alone in embracing neon’s ability to spread messages effectively. By the middle of the century, the lighting was being adopted for more political purposes. “In the 1960s, the Soviets deployed a vast ‘neonization’ of the Eastern bloc capitals to emulate capitalist metropolises,” de Miranda writes. “Because consumer shops were rare in the Polish capital [of Warsaw], they did not hesitate to illuminate the façades of public buildings.” In other words, as opposed to the sole use of the more obvious forms of propaganda via posters or slogans, the mass introduction of neon lighting was a way of getting citizens of Communist cities to see their surroundings with the pizzazz and nighttime glamour of major Western capitals. </p>
<p> De Miranda points out that businesses weren’t alone in embracing neon’s ability to
spread messages effectively. By the middle of the century, the lighting was being
adopted for more political purposes. “In the 1960s, the Soviets deployed a vast
‘neonization’ of the Eastern bloc capitals to emulate capitalist metropolises,” de
Miranda writes. “Because consumer shops were rare in the Polish capital [of Warsaw],
they did not hesitate to illuminate the façades of public buildings.” In other
words, as opposed to the sole use of the more obvious forms of propaganda via
posters or slogans, the mass introduction of neon lighting was a way of getting
citizens of Communist cities to see their surroundings with the pizzazz and
nighttime glamour of major Western capitals. </p>
</section>
<section id="article-section-4">
<p> Neon, around this time, began to be phased out, thanks to cheaper and less labor-intensive alternatives. In addition, the global economic downturn of the 1970s yielded a landscape in which older, flickering neon signs, which perhaps their owners couldn’t afford to fix or replace, came to look like symbols of decline. Where such signs were once sophisticated and novel, they now seemed dated and even seedy. </p>
<p> Neon, around this time, began to be phased out, thanks to cheaper and less
labor-intensive alternatives. In addition, the global economic downturn of the 1970s
yielded a landscape in which older, flickering neon signs, which perhaps their
owners couldn’t afford to fix or replace, came to look like symbols of decline.
Where such signs were once sophisticated and novel, they now seemed dated and even
seedy. </p>
<section>
<h2> Cities are changing fast. Keep up with the <b>CityLab Daily</b> newsletter. </h2><label for="promo-email-input-email">The best way to follow issues you care about.</label>
<h2> Cities are changing fast. Keep up with the <b>CityLab Daily</b> newsletter.
</h2><label for="promo-email-input-email">The best way to follow issues you care
about.</label>
</section>
<p> De Miranda understands this evolution by zooming out and looking at the 1900s as the “neon century.” The author draws a parallel between the physical form of neon lights, which again are essentially containers for electrified gases, and that of a glass capsule—suggesting they are a kind of message in a bottle from a time before the First World War. “Since then, [neon lights] have witnessed all the transformations that have created the world we live in,” de Miranda writes. “Today, they sometimes seem to maintain a hybrid status, somewhere between junkyards and museums, not unlike European capitals themselves.” </p>
<p> De Miranda understands this evolution by zooming out and looking at the 1900s as the
“neon century.” The author draws a parallel between the physical form of neon
lights, which again are essentially containers for electrified gases, and that of a
glass capsule—suggesting they are a kind of message in a bottle from a time before
the First World War. “Since then, [neon lights] have witnessed all the
transformations that have created the world we live in,” de Miranda writes. “Today,
they sometimes seem to maintain a hybrid status, somewhere between junkyards and
museums, not unlike European capitals themselves.” </p>
<figure>
<picture><img alt="" data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg" src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg" /></picture>
<figcaption class="caption"> Martin Wartman, a student at Northern Kentucky University, works on a neon sign at the Neonworks of Cincinnati workshop connected to the American Sign Museum, in 2016. (John Minchillo / AP) </figcaption>
<picture><img alt=""
data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg"
src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg"/>
</picture>
<figcaption class="caption"> Martin Wartman, a student at Northern Kentucky
University, works on a neon sign at the Neonworks of Cincinnati workshop
connected to the American Sign Museum, in 2016. (John Minchillo / AP)
</figcaption>
</figure>
<p> Another mark of neon’s hybridity: Its obsolescence started just as some contemporary artists began using the lights in their sculptures. Bruce Nauman’s 1968 work <em><a href="https://www.stedelijk.nl/en/collection/1097-bruce-nauman-my-name-as-though-it-were-written-on-the-surface-of-the-moon">My Name as Though It Were Written on the Surface of the Moon</a></em> poked fun at the space race—another symbol of 20th-century technological innovation whose moment has passed. The piece uses blue “neon” letters (mercury, actually) to spell out the name “bruce” in lowercase cursive, with each character repeated several times as if to convey a person speaking slowly in outer space. The British artist Tracey Emin has made <a href="https://www.artsy.net/collection/tracey-emin-neon-sculptures-and-prints">sculptures</a> that resemble neon Valentine’s Day candies: They read as garish and sentimental confections with pink, heart-shaped frames that surround blue text fragments. Drawing on the nostalgia-inducing quality of neon, the sculptures’ messages are redolent of old-fashioned movie dialogue, with titles such as “You Loved Me Like a Distant Star” and “The Kiss Was Beautiful.” </p>
<p> Seeing neon lighting tamed in the context of a gallery display fits comfortably with de Miranda’s notion that neon technology is like a time capsule from another age. In museums, works of neon art and design coexist with objects that were ahead of their own time in years past—a poignant fate for a technology that made its name advertising “The World of Tomorrow.” Yet today neon is also experiencing a kind of craft revival. The fact that it can’t be mass-produced has made its fabrication something akin to a cherished artisanal technique. Bars and restaurants hire firms such as Let There Be Neon in Manhattan, or <a href="https://www.instagram.com/theneonqueen/">the L.A.-based master neon artist Lisa Schulte</a>, to create custom signs and works of art. Neon’s story even continues to glow from inside museums such as California’s <a href="https://www.neonmona.org/">Museum of Neon Art</a> and the Neon Museum in Las Vegas. If it can still be a vital medium for artists and designers working today, “neonness” need not only be trapped in the past. It might also capture the mysterious glow of the near future—just as it did a century ago. </p>
<p> Another mark of neon’s hybridity: Its obsolescence started just as some contemporary
artists began using the lights in their sculptures. Bruce Nauman’s 1968 work <em><a
href="https://www.stedelijk.nl/en/collection/1097-bruce-nauman-my-name-as-though-it-were-written-on-the-surface-of-the-moon">My
Name as Though It Were Written on the Surface of the Moon</a></em> poked fun at
the space race—another symbol of 20th-century technological innovation whose moment
has passed. The piece uses blue “neon” letters (mercury, actually) to spell out the
name “bruce” in lowercase cursive, with each character repeated several times as if
to convey a person speaking slowly in outer space. The British artist Tracey Emin
has made <a
href="https://www.artsy.net/collection/tracey-emin-neon-sculptures-and-prints">sculptures</a>
that resemble neon Valentine’s Day candies: They read as garish and sentimental
confections with pink, heart-shaped frames that surround blue text fragments.
Drawing on the nostalgia-inducing quality of neon, the sculptures’ messages are
redolent of old-fashioned movie dialogue, with titles such as “You Loved Me Like a
Distant Star” and “The Kiss Was Beautiful.” </p>
<p> Seeing neon lighting tamed in the context of a gallery display fits comfortably with
de Miranda’s notion that neon technology is like a time capsule from another age. In
museums, works of neon art and design coexist with objects that were ahead of their
own time in years past—a poignant fate for a technology that made its name
advertising “The World of Tomorrow.” Yet today neon is also experiencing a kind of
craft revival. The fact that it can’t be mass-produced has made its fabrication
something akin to a cherished artisanal technique. Bars and restaurants hire firms
such as Let There Be Neon in Manhattan, or <a
href="https://www.instagram.com/theneonqueen/">the L.A.-based master neon
artist Lisa Schulte</a>, to create custom signs and works of art. Neon’s story
even continues to glow from inside museums such as California’s <a
href="https://www.neonmona.org/">Museum of Neon Art</a> and the Neon Museum
in Las Vegas. If it can still be a vital medium for artists and designers working
today, “neonness” need not only be trapped in the past. It might also capture the
mysterious glow of the near future—just as it did a century ago. </p>
<p>
<em>This article originally appeared on <a href="https://www.theatlantic.com/entertainment/archive/2019/04/being-and-neonness-neon-lights-symbol-20th-century/588184/">The Atlantic</a>.</em>
<em>This article originally appeared on <a
href="https://www.theatlantic.com/entertainment/archive/2019/04/being-and-neonness-neon-lights-symbol-20th-century/588184/">The
Atlantic</a>.</em>
</p>
</section>
<section data-include="css:https://cdn.citylab.com/static/a/frontend/dist/citylab/css/components/author-article.cf4e8e0b143f.css">
<section
data-include="css:https://cdn.citylab.com/static/a/frontend/dist/citylab/css/components/author-article.cf4e8e0b143f.css">
<h4> About the Author </h4>
<div itemprop="author" itemscope="itemscope" itemtype="http://schema.org/Person">
<h5 itemprop="name">
<a href="https://www.citylab.com/authors/sarah-archer/">Sarah Archer</a>
</h5>
<p itemprop="description">
<a href="https://www.citylab.com/authors/sarah-archer/" data-omni-click="inherit">Sarah Archer</a> is the author of <em>The Midcentury Kitchen</em>.
<a href="https://www.citylab.com/authors/sarah-archer/"
data-omni-click="inherit">Sarah Archer</a> is the author of <em>The
Midcentury Kitchen</em>.
</p>
</div>
</section>

1834
app/src/main/assets/readability/test/test-pages/citylab-1/source.html
View File

File diff suppressed because one or more lines are too long

1156
app/src/main/assets/readability/test/test-pages/clean-links/expected.html
View File

File diff suppressed because it is too large Load Diff

3954
app/src/main/assets/readability/test/test-pages/clean-links/source.html
View File

File diff suppressed because it is too large Load Diff

72
app/src/main/assets/readability/test/test-pages/cnet-svg-classes/expected.html
View File

File diff suppressed because one or more lines are too long

1547
app/src/main/assets/readability/test/test-pages/cnet-svg-classes/source.html
View File

File diff suppressed because one or more lines are too long

93
app/src/main/assets/readability/test/test-pages/cnet/expected.html
View File

@ -1,29 +1,86 @@
<div id="readability-page-1" class="page">
<div itemprop="articleBody" data-component="lazyloadImages">
<figure section="shortcodeImage"><span><span itemprop="image" itemscope="" itemtype="https://schema.org/ImageObject"><img src="https://cnet1.cbsistatic.com/img/nAMdBzIE1ogVw5bOBZBaiJCt3Ro=/570x0/2014/03/21/863df5d9-e8b8-4b38-851b-5e3f77f2cf0e/mark-zuckerberg-facebook-home-10671610x407.jpg" alt="" width="570" height="0" />
<meta itemprop="url" content="https://cnet1.cbsistatic.com/img/nAMdBzIE1ogVw5bOBZBaiJCt3Ro=/570x0/2014/03/21/863df5d9-e8b8-4b38-851b-5e3f77f2cf0e/mark-zuckerberg-facebook-home-10671610x407.jpg" />
<meta itemprop="height" content="0" />
<meta itemprop="width" content="570" />
<figure section="shortcodeImage"><span><span itemprop="image" itemscope=""
itemtype="https://schema.org/ImageObject"><img
src="https://cnet1.cbsistatic.com/img/nAMdBzIE1ogVw5bOBZBaiJCt3Ro=/570x0/2014/03/21/863df5d9-e8b8-4b38-851b-5e3f77f2cf0e/mark-zuckerberg-facebook-home-10671610x407.jpg"
alt="" width="570" height="0"/>
<meta itemprop="url"
content="https://cnet1.cbsistatic.com/img/nAMdBzIE1ogVw5bOBZBaiJCt3Ro=/570x0/2014/03/21/863df5d9-e8b8-4b38-851b-5e3f77f2cf0e/mark-zuckerberg-facebook-home-10671610x407.jpg"/>
<meta itemprop="height" content="0"/>
<meta itemprop="width" content="570"/>
</span></span>
<figcaption><span class="caption">
<p>Facebook CEO Mark Zuckerberg, the man with the acquisition plan.</p>
</span><span>Photo by James Martin/CNET </span></figcaption>
</figure>
<p>Anyone who has ever been involved in closing a billion-dollar acquisition deal will tell you that you don't go in without a clear, well thought out plan.</p>
<p>Facebook CEO Mark Zuckerberg knows a thing or two about how to seal the deal on blockbuster buys. After all, he's the man behind his company's <a href="https://www.cnet.com/news/facebook-closes-19-billion-deal-for-whatsapp/" target="_blank">$19 billion acquisition</a> of WhatsApp, he <a href="https://www.cnet.com/news/zuckerberg-did-1-billion-instagram-deal-on-his-own/" target="_blank">personally brokered</a> its $1 billion buyout of <a href="https://www.cnet.com/news/why-facebook-plunked-down-1-billion-to-buy-instagram/" target="_blank">Instagram</a> and closed the <a href="https://www.cnet.com/news/facebook-to-buy-oculus-for-2-billion/" target="_blank">$3 billion deal</a> to buy Oculus VR.</p>
<p>Zuckerberg offered a primer on the strategies he and his company employ when they see an attractive target during testimony Tuesday <a href="https://www.cnet.com/news/zenimax-sues-oculus-over-virtual-reality-rift-tech/">in a lawsuit with ZeniMax Media</a>, which accuses Oculus and Facebook of "misappropriating" trade secrets and copyright infringement. At the heart of the lawsuit is technology that helped create liftoff for virtual reality, one of the <a href="http://www.cbsnews.com/videos/the-reality-of-the-virtual-world/" target="_blank" data-component="externalLink">hottest gadget trends today.</a></p>
<p>A key Facebook approach is building a long-term relationship with your target, Zuckerberg said at the trial. These deals don't just pop up over night, he said according to a transcript reviewed by <a href="http://www.businessinsider.com/mark-zuckerberg-explains-facebooks-acquisition-strategy-2017-1" target="_blank" data-component="externalLink">Business Insider</a>. They take time to cultivate. </p>
<p>Anyone who has ever been involved in closing a billion-dollar acquisition deal will tell
you that you don't go in without a clear, well thought out plan.</p>
<p>Facebook CEO Mark Zuckerberg knows a thing or two about how to seal the deal on
blockbuster buys. After all, he's the man behind his company's <a
href="https://www.cnet.com/news/facebook-closes-19-billion-deal-for-whatsapp/"
target="_blank">$19 billion acquisition</a> of WhatsApp, he <a
href="https://www.cnet.com/news/zuckerberg-did-1-billion-instagram-deal-on-his-own/"
target="_blank">personally brokered</a> its $1 billion buyout of <a
href="https://www.cnet.com/news/why-facebook-plunked-down-1-billion-to-buy-instagram/"
target="_blank">Instagram</a> and closed the <a
href="https://www.cnet.com/news/facebook-to-buy-oculus-for-2-billion/"
target="_blank">$3 billion deal</a> to buy Oculus VR.</p>
<p>Zuckerberg offered a primer on the strategies he and his company employ when they see an
attractive target during testimony Tuesday <a
href="https://www.cnet.com/news/zenimax-sues-oculus-over-virtual-reality-rift-tech/">in
a lawsuit with ZeniMax Media</a>, which accuses Oculus and Facebook of
"misappropriating" trade secrets and copyright infringement. At the heart of the lawsuit
is technology that helped create liftoff for virtual reality, one of the <a
href="http://www.cbsnews.com/videos/the-reality-of-the-virtual-world/"
target="_blank" data-component="externalLink">hottest gadget trends today.</a>
</p>
<p>A key Facebook approach is building a long-term relationship with your target, Zuckerberg
said at the trial. These deals don't just pop up over night, he said according to a
transcript reviewed by <a
href="http://www.businessinsider.com/mark-zuckerberg-explains-facebooks-acquisition-strategy-2017-1"
target="_blank" data-component="externalLink">Business Insider</a>. They take
time to cultivate. </p>
<blockquote>
<p>I've been building relationships, at least in Instagram and the WhatsApp cases, for years with the founders and the people that are involved in these companies, which made [it] so that when it became time or when we thought it was the right time to move, we felt like we had a good amount of context and had good relationships so that we could move quickly, which was competitively important and why a lot of these acquisitions, I think, came to us instead of our competitors and ended up being very good acquisitions over time that a lot of competitors wished they had gotten instead. </p>
<p>I've been building relationships, at least in Instagram and the WhatsApp cases, for
years with the founders and the people that are involved in these companies, which
made [it] so that when it became time or when we thought it was the right time to
move, we felt like we had a good amount of context and had good relationships so
that we could move quickly, which was competitively important and why a lot of these
acquisitions, I think, came to us instead of our competitors and ended up being very
good acquisitions over time that a lot of competitors wished they had gotten
instead. </p>
</blockquote>
<p> He also stressed the need assure your target that you have a shared vision about how you will collaborate after the deal is put to bed. Zuckerberg said this was reason Facebook was able to acquire Oculus for less than its original $4 billion asking price.</p>
<blockquote>If this [deal] is going to happen, it's not going to be because we offer a lot of money, although we're going to have to offer a fair price for the company that is more than what they felt like they could do on their own. But they also need to feel like this was actually going to help their mission.</blockquote>
<p>When that doesn't work, Zuckerberg said scare tactics is an effective, if undesirable, way of persuading small startups that they face a better chance of survival if they have Facebook to guide their way rather than going it alone.</p>
<blockquote>That's less my thing, but I think if you are trying to help convince people that they want to join you, helping them understand all the pain that they would have to go through to build it out independently is a valuable tactic. </blockquote>
<p>It also pays to be weary of competing suitors for your startup, Zuckerberg said, and be willing to move fast to stave off rivals and get the deal done.</p>
<blockquote>Often, if a company knows we're offering something, they will offer more. So being able to move quickly not only increases our chance of being able to get a deal done if we want to, but it makes it so we don't have end up having to pay a lot more because the process drags out.</blockquote>
<p>It wasn't clear why these strategies didn't work on Snapchat CEO Evan Spiegel, who <a href="https://www.cnet.com/news/snapchat-said-to-rebuff-3-billion-offer-from-facebook/">famously rebuffed</a> a $3 billion takeover offer from Facebook in 2013.</p>
<p><em><strong>Tech Enabled:</strong> CNET chronicles tech's role in providing new kinds of accessibility. Check it out <a href="https://www.cnet.com/tech-enabled/">here</a>.</em><em><strong><br /></strong></em></p>
<p><em><strong>Technically Literate:</strong> Original works of short fiction with unique perspectives on tech, exclusively on CNET. <a href="https://www.cnet.com/technically-literate/">Here</a>.</em></p>
<p> He also stressed the need assure your target that you have a shared vision about how you
will collaborate after the deal is put to bed. Zuckerberg said this was reason Facebook
was able to acquire Oculus for less than its original $4 billion asking price.</p>
<blockquote>If this [deal] is going to happen, it's not going to be because we offer a lot
of money, although we're going to have to offer a fair price for the company that is
more than what they felt like they could do on their own. But they also need to feel
like this was actually going to help their mission.
</blockquote>
<p>When that doesn't work, Zuckerberg said scare tactics is an effective, if undesirable,
way of persuading small startups that they face a better chance of survival if they have
Facebook to guide their way rather than going it alone.</p>
<blockquote>That's less my thing, but I think if you are trying to help convince people that
they want to join you, helping them understand all the pain that they would have to go
through to build it out independently is a valuable tactic.
</blockquote>
<p>It also pays to be weary of competing suitors for your startup, Zuckerberg said, and be
willing to move fast to stave off rivals and get the deal done.</p>
<blockquote>Often, if a company knows we're offering something, they will offer more. So
being able to move quickly not only increases our chance of being able to get a deal
done if we want to, but it makes it so we don't have end up having to pay a lot more
because the process drags out.
</blockquote>
<p>It wasn't clear why these strategies didn't work on Snapchat CEO Evan Spiegel, who <a
href="https://www.cnet.com/news/snapchat-said-to-rebuff-3-billion-offer-from-facebook/">famously
rebuffed</a> a $3 billion takeover offer from Facebook in 2013.</p>
<p><em><strong>Tech Enabled:</strong> CNET chronicles tech's role in providing new kinds of
accessibility. Check it out <a
href="https://www.cnet.com/tech-enabled/">here</a>.</em><em><strong><br/></strong></em>
</p>
<p><em><strong>Technically Literate:</strong> Original works of short fiction with unique
perspectives on tech, exclusively on CNET. <a
href="https://www.cnet.com/technically-literate/">Here</a>.</em></p>
</div>
</div>

3329
app/src/main/assets/readability/test/test-pages/cnet/source.html
View File

File diff suppressed because one or more lines are too long

52
app/src/main/assets/readability/test/test-pages/cnn/expected.html
View File

@ -1,18 +1,35 @@
<div id="readability-page-1" class="page">
<div id="storytext">
<h2>The U.S. has long been heralded as a land of opportunity -- a place where anyone can succeed regardless of the economic class they were born into.</h2>
<p> But a new report released on Monday by <a href="http://web.stanford.edu/group/scspi-dev/cgi-bin/" target="_blank">Stanford University's Center on Poverty and Inequality</a> calls that into question. </p>
<p> The report assessed poverty levels, income and wealth inequality, economic mobility and unemployment levels among 10 wealthy countries with social welfare programs. </p>
<h2>The U.S. has long been heralded as a land of opportunity -- a place where anyone can
succeed regardless of the economic class they were born into.</h2>
<p> But a new report released on Monday by <a
href="http://web.stanford.edu/group/scspi-dev/cgi-bin/" target="_blank">Stanford
University's Center on Poverty and Inequality</a> calls that into question. </p>
<p> The report assessed poverty levels, income and wealth inequality, economic mobility and
unemployment levels among 10 wealthy countries with social welfare programs. </p>
<div id="smartassetcontainer">
<p> Powered by SmartAsset.com </p>
</div>
<p> Among its key findings: the class you're born into matters much more in the U.S. than many of the other countries. </p>
<p> As the <a href="http://web.stanford.edu/group/scspi-dev/cgi-bin/publications/state-union-report" target="_blank">report states</a>: "[T]he birth lottery matters more in the U.S. than in most well-off countries." </p>
<p> But this wasn't the only finding that suggests the U.S. isn't quite living up to its reputation as a country where everyone has an equal chance to get ahead through sheer will and hard work. </p>
<p> <a href="http://money.cnn.com/2016/01/11/news/economy/rich-taxes/index.html?iid=EL"><span>Related: Rich are paying more in taxes but not as much as they used to</span></a> </p>
<p> The report also suggested the U.S. might not be the "jobs machine" it thinks it is, when compared to other countries. </p>
<p> It ranked near the bottom of the pack based on the levels of unemployment among men and women of prime working age. The study determined this by taking the ratio of employed men and women between the ages of 25 and 54 compared to the total population of each country. </p>
<p> The overall rankings of the countries were as follows:<span> <br />1. Finland <span> <br />2. Norway<span> <br />3. Australia <span> <br />4. Canada<span> <br />5. Germany<span> <br />6. France<span> <br />7. United Kingdom <span> <br />8. Italy<span> <br />9. Spain<span> <br />10. United States </span></span>
<p> Among its key findings: the class you're born into matters much more in the U.S. than
many of the other countries. </p>
<p> As the <a
href="http://web.stanford.edu/group/scspi-dev/cgi-bin/publications/state-union-report"
target="_blank">report states</a>: "[T]he birth lottery matters more in the U.S.
than in most well-off countries." </p>
<p> But this wasn't the only finding that suggests the U.S. isn't quite living up to its
reputation as a country where everyone has an equal chance to get ahead through sheer
will and hard work. </p>
<p>
<a href="http://money.cnn.com/2016/01/11/news/economy/rich-taxes/index.html?iid=EL"><span>Related: Rich are paying more in taxes but not as much as they used to</span></a>
</p>
<p> The report also suggested the U.S. might not be the "jobs machine" it thinks it is, when
compared to other countries. </p>
<p> It ranked near the bottom of the pack based on the levels of unemployment among men and
women of prime working age. The study determined this by taking the ratio of employed
men and women between the ages of 25 and 54 compared to the total population of each
country. </p>
<p> The overall rankings of the countries were as
follows:<span> <br/>1. Finland <span> <br/>2. Norway<span> <br/>3. Australia <span> <br/>4. Canada<span> <br/>5. Germany<span> <br/>6. France<span> <br/>7. United Kingdom <span> <br/>8. Italy<span> <br/>9. Spain<span> <br/>10. United States </span></span>
</span>
</span>
</span>
@ -22,9 +39,16 @@
</span>
</span>
</p>
<p> The low ranking the U.S. received was due to its extreme levels of wealth and income inequality and the ineffectiveness of its "safety net" -- social programs aimed at reducing poverty. </p>
<p> <a href="http://money.cnn.com/2016/01/05/news/economy/chicago-segregated/index.html?iid=EL"><span>Related: Chicago is America's most segregated city</span></a> </p>
<p> The report concluded that the American safety net was ineffective because it provides only half the financial help people need. Additionally, the levels of assistance in the U.S. are generally lower than in other countries. </p>
<p><span> CNNMoney (New York) </span> <span>First published February 1, 2016: 1:28 AM ET</span> </p>
<p> The low ranking the U.S. received was due to its extreme levels of wealth and income
inequality and the ineffectiveness of its "safety net" -- social programs aimed at
reducing poverty. </p>
<p>
<a href="http://money.cnn.com/2016/01/05/news/economy/chicago-segregated/index.html?iid=EL"><span>Related: Chicago is America's most segregated city</span></a>
</p>
<p> The report concluded that the American safety net was ineffective because it provides
only half the financial help people need. Additionally, the levels of assistance in the
U.S. are generally lower than in other countries. </p>
<p><span> CNNMoney (New York) </span>
<span>First published February 1, 2016: 1:28 AM ET</span></p>
</div>
</div>

2466
app/src/main/assets/readability/test/test-pages/cnn/source.html
View File

File diff suppressed because it is too large Load Diff

17
app/src/main/assets/readability/test/test-pages/comment-inside-script-parsing/expected.html
View File

@ -1,11 +1,18 @@
<div id="readability-page-1" class="page">
<div>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui
officia deserunt mollit anim id est laborum.</p>
</div>
<div>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat
nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui
officia deserunt mollit anim id est laborum.</p>
</div>
</div>

45
app/src/main/assets/readability/test/test-pages/comment-inside-script-parsing/source.html
View File

@ -1,34 +1,35 @@
<html>
<head><title>Test script parsing</title></head>
<head><title>Test script parsing</title></head>
<body>
<script>
<script>
<!--
Silly test
<script src="foo.js"></script>
-->
</script>
<article>
<script src="foo.js">
</script>
-->
</script>
<article>
<h1>Lorem</h1>
<div>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
</div>
<h2>Foo</h2>
<div>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
<p>Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
</div>
</article>
</article>
</body>
</html>

34
app/src/main/assets/readability/test/test-pages/daringfireball-1/expected.html
View File

@ -3,8 +3,11 @@
<h2>About This Site</h2>
<p>Daring Fireball is written and produced by John Gruber.</p>
<p>
<a href="http://fakehost/graphics/author/addison-bw.jpg"> <img src="http://fakehost/graphics/author/addison-bw-425.jpg" alt="Photograph of the author." /></a>
<br /><em>Portrait by <a href="http://superbiate.com/inquiries/">George Del Barrio</a></em>
<a href="http://fakehost/graphics/author/addison-bw.jpg"> <img
src="http://fakehost/graphics/author/addison-bw-425.jpg"
alt="Photograph of the author."/></a>
<br/><em>Portrait by <a href="http://superbiate.com/inquiries/">George Del
Barrio</a></em>
</p>
<h2>Mac Apps</h2>
<ul>
@ -24,10 +27,29 @@
</ul>
<h2>Server Software</h2>
<p>The Daring Fireball website is hosted by <a href="http://joyent.com/">Joyent</a>.</p>
<p>Articles and links are published through <a href="http://movabletype.org/">Movable Type</a>. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s <a href="http://bradchoate.com/weblog/2003/06/24/regular-expressions">MT-Regex</a> and <a href="http://bradchoate.com/weblog/2004/10/20/mtifempty">MT-IfEmpty</a>, and <a href="http://bumppo.net/projects/amputator/">Nat Irons’s Amputator</a>.</p>
<p>Stats are tracked using <a href="http://haveamint.com/">Mint</a>. Additional web nerdery, including the membership system, is fueled by <a href="http://perl.org/">Perl</a>, <a href="http://www.php.net/">PHP</a>, and <a href="http://www.mysql.com/">MySQL</a>.</p>
<p>Articles and links are published through <a href="http://movabletype.org/">Movable
Type</a>. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses
several excellent Movable Type plug-ins, including Brad Choate’s <a
href="http://bradchoate.com/weblog/2003/06/24/regular-expressions">MT-Regex</a>
and <a href="http://bradchoate.com/weblog/2004/10/20/mtifempty">MT-IfEmpty</a>, and <a
href="http://bumppo.net/projects/amputator/">Nat Irons’s Amputator</a>.</p>
<p>Stats are tracked using <a href="http://haveamint.com/">Mint</a>. Additional web nerdery,
including the membership system, is fueled by <a href="http://perl.org/">Perl</a>, <a
href="http://www.php.net/">PHP</a>, and <a
href="http://www.mysql.com/">MySQL</a>.</p>
<h2>Web Standards</h2>
<p>Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either <a href="http://www.whatwg.org/specs/web-apps/current-work/">HTML 5</a> or XHTML 4.01 Transitional, its layout is constructed using <a href="http://jigsaw.w3.org/css-validator/validator?uri=http://daringfireball.net/css/fireball_screen.css">valid CSS</a>, and its syndicated feed is <a href="http://feedvalidator.org/check?url=http%3A%2F%2Fdaringfireball.net%2Findex.xml">valid Atom</a>.</p>
<p>If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.</p>
<p>Web standards are important, and Daring Fireball adheres to them. Specifically, Daring
Fireball’s HTML markup should validate as either <a
href="http://www.whatwg.org/specs/web-apps/current-work/">HTML 5</a> or XHTML
4.01 Transitional, its layout is constructed using <a
href="http://jigsaw.w3.org/css-validator/validator?uri=http://daringfireball.net/css/fireball_screen.css">valid
CSS</a>, and its syndicated feed is <a
href="http://feedvalidator.org/check?url=http%3A%2F%2Fdaringfireball.net%2Findex.xml">valid
Atom</a>.</p>
<p>If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that
doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you
complain about this, I will laugh at you, because I do not care. If, however, you are
using a modern, standards-compliant browser and have trouble viewing or reading Daring
Fireball, please do let me know.</p>
</div>
</div>

205
app/src/main/assets/readability/test/test-pages/daringfireball-1/source.html
View File

@ -1,4 +1,5 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
@ -13,20 +14,22 @@
<link rel="alternate" type="application/atom+xml" href="/feeds/main"/>
<script src="/js/js-global/FancyZoom.js" type="text/javascript"></script>
<script src="/js/js-global/FancyZoomHTML.js" type="text/javascript"></script>
<link rel="shortcut icon" href="/favicon.ico"/> </head>
<link rel="shortcut icon" href="/favicon.ico"/>
</head>
<body onload="setupZoom()">
<div id="Box">
<div id="Banner">
<a href="/" title="Daring Fireball: Home"><img src="/graphics/logos/" alt="Daring Fireball" height="56"/></a>
</div>
<div id="Sidebar">
<p>By <strong>John&nbsp;Gruber</strong></p>
<ul>
<!--&#9733;-->
<li><a href="/archive/" title="Previous articles.">Archive</a></li>
<li>
<script type="text/javascript">
<div id="Box">
<div id="Banner">
<a href="/" title="Daring Fireball: Home"><img src="/graphics/logos/" alt="Daring Fireball"
height="56"/></a>
</div>
<div id="Sidebar">
<p>By <strong>John&nbsp;Gruber</strong></p>
<ul>
<!--&#9733;-->
<li><a href="/archive/" title="Previous articles.">Archive</a></li>
<li>
<script type="text/javascript">
// <![CDATA[
function ReadCookie(name) {
var nameEQ = name + "=";
@ -47,63 +50,95 @@
// Default to not putting separate LL item in sidebar:
}
// ]]>
</script>
</li>
<li><a href="/thetalkshow/" title="The world’s most popular podcast.">The Talk Show</a></li>
<li><a href="http://vesperapp.co/" title="A simple, elegant iOS app for collecting notes, ideas, thoughts, and images.">Vesper: <em>Now for iPad</em></a></li>
<li><a href="/projects/" title="Software projects, including SmartyPants and Markdown.">Projects</a></li>
<li><a href="/contact/" title="How to send email regarding Daring Fireball.">Contact</a></li>
<li><a href="/colophon/" title="About this site and the tools used to produce it.">Colophon</a></li>
<li><a href="/feeds/">RSS Feed</a></li>
<li><a href="https://twitter.com/daringfireball">Twitter</a></li>
<li><a href="/feeds/sponsors/">Sponsorship</a></li>
</script>
</li>
<li><a href="/thetalkshow/" title="The world’s most popular podcast.">The Talk Show</a>
</li>
<li><a href="http://vesperapp.co/"
title="A simple, elegant iOS app for collecting notes, ideas, thoughts, and images.">Vesper:
<em>Now for iPad</em></a></li>
<li><a href="/projects/" title="Software projects, including SmartyPants and Markdown.">Projects</a>
</li>
<li><a href="/contact/" title="How to send email regarding Daring Fireball.">Contact</a>
</li>
<li><a href="/colophon/" title="About this site and the tools used to produce it.">Colophon</a>
</li>
<li><a href="/feeds/">RSS Feed</a></li>
<li><a href="https://twitter.com/daringfireball">Twitter</a></li>
<li><a href="/feeds/sponsors/">Sponsorship</a></li>
</ul>
</div>
<!-- Sidebar -->
<div id="Main">
<div class="article">
<h1>About This Site</h1>
<p>Daring Fireball is written and produced by John Gruber.</p>
<p>
<a href="/graphics/author/addison-bw.jpg"> <img
src="/graphics/author/addison-bw-425.jpg" alt="Photograph of the author."
style="border: 0;"/></a>
<br/><em>Portrait by <a href="http://superbiate.com/inquiries/">George Del
Barrio</a></em></p>
<h2>Mac Apps</h2>
<ul>
<li><a href="http://www.barebones.com/products/bbedit/">BBEdit</a></li>
<li><a href="http://www.flyingmeat.com/acorn/">Acorn</a></li>
<li><a href="http://www.red-sweater.com/marsedit/">MarsEdit</a></li>
<li><a href="http://aged-and-distilled.com/napkin/">Napkin</a></li>
<li><a href="http://www.barebones.com/products/Yojimbo/">Yojimbo</a></li>
<li><a href="http://www.panic.com/transmit/">Transmit</a></li>
<li><a href="http://latenightsw.com/sd4/index.html">Script Debugger</a></li>
<li><a href="http://www.ambrosiasw.com/utilities/snapzprox/">Snapz Pro X</a></li>
<li><a href="http://nightly.webkit.org/">WebKit</a></li>
</ul>
<h2>iPhone Apps</h2>
<ul>
<li><a href="http://vesperapp.co/">Vesper</a></li>
</ul>
<h2>Server Software</h2>
<p>The Daring Fireball website is hosted by <a href="http://joyent.com/">Joyent</a>.</p>
<p>Articles and links are published through <a href="http://movabletype.org/">Movable
Type</a>. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball
uses several excellent Movable Type plug-ins, including Brad Choate’s <a
href="http://bradchoate.com/weblog/2003/06/24/regular-expressions">MT-Regex</a>
and <a href="http://bradchoate.com/weblog/2004/10/20/mtifempty">MT-IfEmpty</a>, and
<a href="http://bumppo.net/projects/amputator/">Nat Irons’s Amputator</a>.</p>
<p>Stats are tracked using <a href="http://haveamint.com/">Mint</a>. Additional web
nerdery, including the membership system, is fueled by <a href="http://perl.org/">Perl</a>,
<a href="http://www.php.net/">PHP</a>, and <a href="http://www.mysql.com/">MySQL</a>.
</p>
<h2>Web Standards</h2>
<p>Web standards are important, and Daring Fireball adheres to them. Specifically,
Daring Fireball’s HTML markup should validate as either <a
href="http://www.whatwg.org/specs/web-apps/current-work/">HTML 5</a> or
XHTML 4.01 Transitional, its layout is constructed using <a
href="http://jigsaw.w3.org/css-validator/validator?uri=http://daringfireball.net/css/fireball_screen.css">valid
CSS</a>, and its syndicated feed is <a
href="http://feedvalidator.org/check?url=http%3A%2F%2Fdaringfireball.net%2Findex.xml">valid
Atom</a>.</p>
<p>If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser
that doesn’t support web standards. Internet Explorer, I’m looking in your
direction. If you complain about this, I will laugh at you, because I do not care.
If, however, you are using a modern, standards-compliant browser and have trouble
viewing or reading Daring Fireball, please do let me know.</p>
</div>
<!-- Sidebar -->
<div id="Main">
<div class="article">
<h1>About This Site</h1>
<p>Daring Fireball is written and produced by John Gruber.</p>
<p>
<a href="/graphics/author/addison-bw.jpg"> <img src="/graphics/author/addison-bw-425.jpg" alt="Photograph of the author." style="border: 0;"/></a>
<br/><em>Portrait by <a href="http://superbiate.com/inquiries/">George Del Barrio</a></em> </p>
<h2>Mac Apps</h2>
<ul>
<li><a href="http://www.barebones.com/products/bbedit/">BBEdit</a></li>
<li><a href="http://www.flyingmeat.com/acorn/">Acorn</a></li>
<li><a href="http://www.red-sweater.com/marsedit/">MarsEdit</a></li>
<li><a href="http://aged-and-distilled.com/napkin/">Napkin</a></li>
<li><a href="http://www.barebones.com/products/Yojimbo/">Yojimbo</a></li>
<li><a href="http://www.panic.com/transmit/">Transmit</a></li>
<li><a href="http://latenightsw.com/sd4/index.html">Script Debugger</a></li>
<li><a href="http://www.ambrosiasw.com/utilities/snapzprox/">Snapz Pro X</a></li>
<li><a href="http://nightly.webkit.org/">WebKit</a></li>
</ul>
<h2>iPhone Apps</h2>
<ul>
<li><a href="http://vesperapp.co/">Vesper</a></li>
</ul>
<h2>Server Software</h2>
<p>The Daring Fireball website is hosted by <a href="http://joyent.com/">Joyent</a>.</p>
<p>Articles and links are published through <a href="http://movabletype.org/">Movable Type</a>. In addition to my own SmartyPants and Markdown plug-ins, Daring Fireball uses several excellent Movable Type plug-ins, including Brad Choate’s <a href="http://bradchoate.com/weblog/2003/06/24/regular-expressions">MT-Regex</a> and <a href="http://bradchoate.com/weblog/2004/10/20/mtifempty">MT-IfEmpty</a>, and <a href="http://bumppo.net/projects/amputator/">Nat Irons’s Amputator</a>.</p>
<p>Stats are tracked using <a href="http://haveamint.com/">Mint</a>. Additional web nerdery, including the membership system, is fueled by <a href="http://perl.org/">Perl</a>, <a href="http://www.php.net/">PHP</a>, and <a href="http://www.mysql.com/">MySQL</a>.</p>
<h2>Web Standards</h2>
<p>Web standards are important, and Daring Fireball adheres to them. Specifically, Daring Fireball’s HTML markup should validate as either <a href="http://www.whatwg.org/specs/web-apps/current-work/">HTML 5</a> or XHTML 4.01 Transitional, its layout is constructed using <a href="http://jigsaw.w3.org/css-validator/validator?uri=http://daringfireball.net/css/fireball_screen.css">valid CSS</a>, and its syndicated feed is <a href="http://feedvalidator.org/check?url=http%3A%2F%2Fdaringfireball.net%2Findex.xml">valid Atom</a>.</p>
<p>If Daring Fireball looks goofy in your browser, you’re likely using a shitty browser that doesn’t support web standards. Internet Explorer, I’m looking in your direction. If you complain about this, I will laugh at you, because I do not care. If, however, you are using a modern, standards-compliant browser and have trouble viewing or reading Daring Fireball, please do let me know.</p>
</div>
<!-- article -->
<div id="Footer">
<form id="SiteSearch" action="https://daringfireball.net/search" method="get" style="margin-bottom: 2.5em;">
<div>
<input name="q" type="text" value="" style="margin-right: 8px; width: 66%;"/>
<input type="submit" value="Search"/> </div>
</form>
<p class="smallprint"> <a href="/preferences/" title="Customize the font size and presentation options for this web site.">Display Preferences</a>
<br/>
<br/> Copyright © 2002–2015 The Daring Fireball Company LLC. </p>
</div>
<div id="SidebarTheDeck">
<script type="text/javascript">
<!-- article -->
<div id="Footer">
<form id="SiteSearch" action="https://daringfireball.net/search" method="get"
style="margin-bottom: 2.5em;">
<div>
<input name="q" type="text" value="" style="margin-right: 8px; width: 66%;"/>
<input type="submit" value="Search"/></div>
</form>
<p class="smallprint"><a href="/preferences/"
title="Customize the font size and presentation options for this web site.">Display
Preferences</a>
<br/>
<br/> Copyright © 2002–2015 The Daring Fireball Company LLC. </p>
</div>
<div id="SidebarTheDeck">
<script type="text/javascript">
// <![CDATA[
(function(id) {
document.write('<script type="text/javascript" src="' + '//connect.decknetwork.net/deck' + id + '_js.php?' + (new Date().getTime()) + '"></' + 'script>');
@ -112,13 +147,17 @@
var sidebarDiv = document.getElementById("Sidebar");
sidebarDiv.appendChild(deckDiv);
// ]]>
</script>
<p id="ViaTheDeck">
<a href="http://decknetwork.net/"> <img src="//daringfireball.net/graphics/madison/via_the_deck.png" alt="Ads via The Deck" class="the_deck_promo" width="70"/> <span>Ads via The Deck</span> </a>
</p>
</div>
<!-- Google Analytics -->
<script type="text/javascript">
</script>
<p id="ViaTheDeck">
<a href="http://decknetwork.net/"> <img
src="//daringfireball.net/graphics/madison/via_the_deck.png"
alt="Ads via The Deck" class="the_deck_promo" width="70"/> <span>Ads via The Deck</span>
</a>
</p>
</div>
<!-- Google Analytics -->
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-593949-1']);
_gaq.push(['_trackPageview']);
@ -130,9 +169,10 @@
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ga, s);
})();
</script>
<!-- Asynchronously load Mint -->
<script type="text/javascript">
</script>
<!-- Asynchronously load Mint -->
<script type="text/javascript">
(function() {
var ma = document.createElement('script');
ma.type = 'text/javascript';
@ -141,11 +181,12 @@
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ma, s);
})();
</script>
</div>
<!-- Main -->
</script>
</div>
<!-- box -->
<!-- Main -->
</div>
<!-- box -->
</body>
</html>

52
app/src/main/assets/readability/test/test-pages/data-url-image/expected.html
View File

File diff suppressed because one or more lines are too long

57
app/src/main/assets/readability/test/test-pages/data-url-image/source.html
View File

File diff suppressed because one or more lines are too long

639
app/src/main/assets/readability/test/test-pages/dropbox-blog/expected.html
View File

@ -1,11 +1,32 @@
<div id="readability-page-1" class="page">
<div>
<div>
<p> I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign. </p>
<p> This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems used by different engineering teams. It would reduce redundant development, incompatibilities, and reliance on legacy software. There were no open-source projects or buy-not-build solutions that worked well for our use case and scale, so we had to create our own. ATF is both an important and interesting challenge, though, so we were happy to design, build and deploy our own in-house service. </p>
<p> ATF not only had to work well, it had to work well at scale: It would be a foundational building block of Dropbox infrastructure. It would need to handle 10,000 async tasks per second from the start, and be architected for future growth. It would need to support nearly 100 unique async task types from the start, again with room to grow. There were at least two dozen engineering teams that would want to use it for entirely different parts of our codebase, for many products and services.&#160; </p>
<p> As any engineer would, we Googled to see what other companies with mega-scale services had done to handle async tasks. We were disappointed to find little material published by engineers who built supersized async services. </p>
<p> Now that ATF is deployed and currently serving 9,000 async tasks scheduled per second and in use by 28 engineering teams internally, we’re glad to fill that information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide for the engineering community seeking their own async solutions. </p>
<p> I joined Dropbox not long after graduating with a Master’s degree in computer
science. Aside from an internship, this was my first big-league engineering job. My
team had already begun designing a critical internal service that most of our
software would use: It would handle asynchronous computing requests behind the
scenes, powering everything from dragging a file into a Dropbox folder to scheduling
a marketing campaign. </p>
<p> This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems
used by different engineering teams. It would reduce redundant development,
incompatibilities, and reliance on legacy software. There were no open-source
projects or buy-not-build solutions that worked well for our use case and scale, so
we had to create our own. ATF is both an important and interesting challenge,
though, so we were happy to design, build and deploy our own in-house service. </p>
<p> ATF not only had to work well, it had to work well at scale: It would be a
foundational building block of Dropbox infrastructure. It would need to handle
10,000 async tasks per second from the start, and be architected for future growth.
It would need to support nearly 100 unique async task types from the start, again
with room to grow. There were at least two dozen engineering teams that would want
to use it for entirely different parts of our codebase, for many products and
services.&#160; </p>
<p> As any engineer would, we Googled to see what other companies with mega-scale
services had done to handle async tasks. We were disappointed to find little
material published by engineers who built supersized async services. </p>
<p> Now that ATF is deployed and currently serving 9,000 async tasks scheduled per
second and in use by 28 engineering teams internally, we’re glad to fill that
information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide
for the engineering community seeking their own async solutions. </p>
</div>
<div>
<p id="introduction">
@ -13,8 +34,15 @@
</p>
</div>
<div>
<p> Scheduling asynchronous tasks on-demand is a critical capability that powers many features and internal platforms at Dropbox. Async Task Framework (ATF) is the infrastructural system that supports this capability at Dropbox through a callback-based architecture. ATF enables developers to define callbacks, and schedule tasks that execute against these pre-defined callbacks. </p>
<p> Since its introduction over a year ago, ATF has gone on to become an important building block in the Dropbox infrastructure, used by nearly 30 internal teams across our codebase. It currently supports 100+ use cases which require either immediate or delayed task scheduling.&#160; </p>
<p> Scheduling asynchronous tasks on-demand is a critical capability that powers many
features and internal platforms at Dropbox. Async Task Framework (ATF) is the
infrastructural system that supports this capability at Dropbox through a
callback-based architecture. ATF enables developers to define callbacks, and
schedule tasks that execute against these pre-defined callbacks. </p>
<p> Since its introduction over a year ago, ATF has gone on to become an important
building block in the Dropbox infrastructure, used by nearly 30 internal teams
across our codebase. It currently supports 100+ use cases which require either
immediate or delayed task scheduling.&#160; </p>
</div>
<div>
<p id="glossary">
@ -22,7 +50,8 @@
</p>
</div>
<div>
<p> Some basic terms repeatedly used in this post, defined as used in the context of this discussion. </p>
<p> Some basic terms repeatedly used in this post, defined as used in the context of
this discussion. </p>
<p>
<b>Lambda:</b> A callback implementing business logic.
</p>
@ -43,16 +72,22 @@
</div>
<div>
<p>
<b>Task scheduling</b><br /> Clients can schedule tasks to execute at a specified time. Tasks can be scheduled for immediate execution, or delayed to fit the use case.
<b>Task scheduling</b><br/> Clients can schedule tasks to execute at a specified
time. Tasks can be scheduled for immediate execution, or delayed to fit the use
case.
</p>
<p>
<b>Priority based execution</b><br /> Tasks should be associated with a priority. Tasks with higher priority should get executed before tasks with a lower priority once they are ready for execution.
<b>Priority based execution</b><br/> Tasks should be associated with a priority.
Tasks with higher priority should get executed before tasks with a lower priority
once they are ready for execution.
</p>
<p>
<b>Task gating</b><br /> ATF enables the the gating of tasks based on lambda, or a subset of tasks on a lambda based on collection. Tasks can be gated to be completely dropped or paused until a suitable time for execution.
<b>Task gating</b><br/> ATF enables the the gating of tasks based on lambda, or a
subset of tasks on a lambda based on collection. Tasks can be gated to be completely
dropped or paused until a suitable time for execution.
</p>
<p>
<b>Track task status</b><br /> Clients can query the status of a scheduled task.
<b>Track task status</b><br/> Clients can query the status of a scheduled task.
</p>
</div>
<div>
@ -62,19 +97,30 @@
</div>
<div>
<p>
<b>At-least once task execution<br /></b> The ATF system guarantees that a task is executed at least once after being scheduled. Execution is said to be complete once the user-defined callback signals task completion to the ATF system.
<b>At-least once task execution<br/></b> The ATF system guarantees that a task is
executed at least once after being scheduled. Execution is said to be complete once
the user-defined callback signals task completion to the ATF system.
</p>
<p>
<b>No concurrent task execution<br /></b> The ATF system guarantees that at most one instance of a task will be actively executing at any given in point. This helps users write their callbacks without designing for concurrent execution of the same task from different locations.
<b>No concurrent task execution<br/></b> The ATF system guarantees that at most one
instance of a task will be actively executing at any given in point. This helps
users write their callbacks without designing for concurrent execution of the same
task from different locations.
</p>
<p>
<b>Isolation<br /></b> Tasks in a given lambda are isolated from the tasks in other lambdas. This isolation spans across several dimensions, including worker capacity for task execution and resource use for task scheduling. Tasks on the same lambda but different priority levels are also isolated in their resource use for task scheduling.
<b>Isolation<br/></b> Tasks in a given lambda are isolated from the tasks in other
lambdas. This isolation spans across several dimensions, including worker capacity
for task execution and resource use for task scheduling. Tasks on the same lambda
but different priority levels are also isolated in their resource use for task
scheduling.
</p>
<p>
<b>Delivery latency<br /></b> 95% of tasks begin execution within five seconds from their scheduled execution time.
<b>Delivery latency<br/></b> 95% of tasks begin execution within five seconds from
their scheduled execution time.
</p>
<p>
<b>High availability for task scheduling<br /></b> The ATF service is 99.9% available to accept task scheduling requests from any client.
<b>High availability for task scheduling<br/></b> The ATF service is 99.9% available
to accept task scheduling requests from any client.
</p>
</div>
<div>
@ -85,13 +131,22 @@
<div>
<p> Following are some restrictions we place on the callback logic (lambda): </p>
<p>
<b>Idempotence</b><br /> A single task on a lambda can be executed multiple times within the ATF system. Developers should ensure that their lambda logic and correctness of task execution in clients are not affected by this.
<b>Idempotence</b><br/> A single task on a lambda can be executed multiple times
within the ATF system. Developers should ensure that their lambda logic and
correctness of task execution in clients are not affected by this.
</p>
<p>
<b>Resiliency</b><br /> Worker processes which execute tasks might die at any point during task execution. ATF retries abruptly interrupted tasks, which could also be retried on different hosts. Lambda owners must design their lambdas such that retries on different hosts do not affect lambda correctness.
<b>Resiliency</b><br/> Worker processes which execute tasks might die at any point
during task execution. ATF retries abruptly interrupted tasks, which could also be
retried on different hosts. Lambda owners must design their lambdas such that
retries on different hosts do not affect lambda correctness.
</p>
<p>
<b>Terminal state handling<br /></b> ATF retries tasks until they are signaled to be complete from the lambda logic. Client code can mark a task as successfully completed, fatally terminated, or retriable. It is critical that lambda owners design clients to signal task completion appropriately to avoid misbehavior such as infinite retries.&#160;
<b>Terminal state handling<br/></b> ATF retries tasks until they are signaled to be
complete from the lambda logic. Client code can mark a task as successfully
completed, fatally terminated, or retriable. It is critical that lambda owners
design clients to signal task completion appropriately to avoid misbehavior such as
infinite retries.&#160;
</p>
</div>
<div>
@ -101,31 +156,53 @@
</div>
<div>
<figure>
<img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x844px-1.png" aria-hidden="false" alt="Async Task Framework (ATF) [Fig 1]" height="1688" width="1440" />
<figcaption> Async Task Framework (ATF) [Fig 1] </figcaption>
<img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x844px-1.png"
aria-hidden="false" alt="Async Task Framework (ATF) [Fig 1]" height="1688"
width="1440"/>
<figcaption> Async Task Framework (ATF) [Fig 1]</figcaption>
</figure>
</div>
<div>
<p> In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.)&#160;In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) Dropbox <a href="https://dropbox.tech/infrastructure/courier-dropbox-migration-to-grpc">uses gRPC</a> for remote calls and our in-house <a href="https://dropbox.tech/infrastructure/reintroducing-edgestore">Edgestore</a> to store tasks. </p>
<p> In this section, we describe the high-level architecture of ATF and give brief
description of its different components. (See Fig. 1 above.)&#160;In this section,
we describe the high-level architecture of ATF and give brief description of its
different components. (See Fig. 1 above.) Dropbox <a
href="https://dropbox.tech/infrastructure/courier-dropbox-migration-to-grpc">uses
gRPC</a> for remote calls and our in-house <a
href="https://dropbox.tech/infrastructure/reintroducing-edgestore">Edgestore</a>
to store tasks. </p>
<p> ATF consists of the following components:&#160; </p>
<ul>
<li>Frontend </li>
<li>Task Store </li>
<li>Store Consumer </li>
<li>Queue </li>
<li>Controller </li>
<li>Executor </li>
<li>Heartbeat and Status Controller (HSC)<span><br /></span>
<li>Frontend</li>
<li>Task Store</li>
<li>Store Consumer</li>
<li>Queue</li>
<li>Controller</li>
<li>Executor</li>
<li>Heartbeat and Status Controller (HSC)<span><br/></span>
</li>
</ul>
<p>
<span><b>Frontend</b><br /> This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.</span><br />
<span><b>Frontend</b><br/> This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.</span><br/>
</p>
<p>
<b>Task Store<br /></b> ATF tasks are stored in and triggered from the task store. The task store could be any generic data store with indexed querying capability. In ATF’s case, We use our in-house metadata store Edgestore to power the task store. More details can be&#160;found in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">D</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">ata</a> <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">M</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">odel</a> section below.
<b>Task Store<br/></b> ATF tasks are stored in and triggered from the task store.
The task store could be any generic data store with indexed querying capability. In
ATF’s case, We use our in-house metadata store Edgestore to power the task store.
More details can be&#160;found in the <a
href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">D</a><a
href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">ata</a>
<a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">M</a><a
href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">odel</a>
section below.
</p>
<p>
<b>Store Consumer<br /></b> The Store Consumer is a service that periodically polls the task store to find tasks that are ready for execution and pushes them onto the right queues, as described in the queue section below. These could be tasks that are newly ready for execution, or older tasks that are ready for execution again because they either failed in a retriable way on execution, or were dropped elsewhere within the ATF system.&#160;
<b>Store Consumer<br/></b> The Store Consumer is a service that periodically polls
the task store to find tasks that are ready for execution and pushes them onto the
right queues, as described in the queue section below. These could be tasks that are
newly ready for execution, or older tasks that are ready for execution again because
they either failed in a retriable way on execution, or were dropped elsewhere within
the ATF system.&#160;
</p>
<p> Below is a simple walkthrough of the Store Consumer’s function:&#160; </p>
</div>
@ -136,16 +213,33 @@
3. update task statuses</code></pre>
</div>
<div>
<p> The Store Consumer polls tasks that failed in earlier execution attempts. This helps with the at-least-once guarantee that the ATF system provides. More details on how the Store Consumer polls new and previously failed tasks is presented in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=342792671048375002388848&amp;h2=Lifecycle-of-a-task">Lifecycle of a task</a> section below. </p>
<p> The Store Consumer polls tasks that failed in earlier execution attempts. This helps
with the at-least-once guarantee that the ATF system provides. More details on how
the Store Consumer polls new and previously failed tasks is presented in the <a
href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=342792671048375002388848&amp;h2=Lifecycle-of-a-task">Lifecycle
of a task</a> section below. </p>
<p>
<b>Queue<br /></b> ATF uses AWS <a href="https://aws.amazon.com/sqs/">Simple Queue Service</a> (SQS) to queue tasks internally. These queues act as a buffer between the Store Consumer and Controllers (described below). Each <span>&lt;lambda, priority&gt;</span> &#160;pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is <span>#lambdas x #priorities</span>.
<b>Queue<br/></b> ATF uses AWS <a href="https://aws.amazon.com/sqs/">Simple Queue
Service</a> (SQS) to queue tasks internally. These queues act as a buffer between
the Store Consumer and Controllers (described below). Each <span>&lt;lambda, priority&gt;</span>
&#160;pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is
<span>#lambdas x #priorities</span>.
</p>
<p>
<b>Controller<br /></b> Worker hosts are physical hosts dedicated for task execution. Each worker host has one controller process responsible for polling tasks from SQS queues in a background thread, and then pushing them onto process local buffered queues. The Controller is only aware of the lambdas it is serving and thus polls only the limited set of necessary queues.&#160;
<b>Controller<br/></b> Worker hosts are physical hosts dedicated for task execution.
Each worker host has one controller process responsible for polling tasks from SQS
queues in a background thread, and then pushing them onto process local buffered
queues. The Controller is only aware of the lambdas it is serving and thus polls
only the limited set of necessary queues.&#160;
</p>
<p> The Controller serves tasks from its process local queue as a response to <span>NextWork</span> RPCs. This is the layer where execution level task prioritization occurs. The Controller has different process level queues for tasks of different priorities and can thus prioritize tasks in response to <span>NextWork</span> RPCs. </p>
<p> The Controller serves tasks from its process local queue as a response to <span>NextWork</span>
RPCs. This is the layer where execution level task prioritization occurs. The
Controller has different process level queues for tasks of different priorities and
can thus prioritize tasks in response to <span>NextWork</span> RPCs. </p>
<p>
<b>Executor<br /></b> The Executor is a process with multiple threads, responsible for the actual task execution. Each thread within an Executor process follows this simple loop:
<b>Executor<br/></b> The Executor is a process with multiple threads, responsible
for the actual task execution. Each thread within an Executor process follows this
simple loop:
</p>
</div>
<div>
@ -154,9 +248,18 @@
do_work(w)</code></pre>
</div>
<div>
<p> Each worker host has a single Controller process and multiple executor processes. Both the Controller and Executors work in a “pull” model, in which active loops continuously long-poll for new work to be done. </p>
<p> Each worker host has a single Controller process and multiple executor processes.
Both the Controller and Executors work in a “pull” model, in which active loops
continuously long-poll for new work to be done. </p>
<p>
<b>Heartbeat and Status Controller (HSC)</b><br /> The HSC serves RPCs for claiming a task for execution (<span>ClaimTask</span>), setting task status after execution (<span>SetResults</span>) and heartbeats during task execution (<span>Heartbeat</span>). <span>ClaimTask</span> requests originate from the Controllers in response to <span>NextWork</span> requests. <span>Heartbeat</span> and <span>SetResults</span> requests originate from executor processes during and after task execution. The HSC interacts with the task store to update the task status on the kind of request it receives.
<b>Heartbeat and Status Controller (HSC)</b><br/> The HSC serves RPCs for claiming a
task for execution (<span>ClaimTask</span>), setting task status after execution
(<span>SetResults</span>) and heartbeats during task execution
(<span>Heartbeat</span>). <span>ClaimTask</span> requests originate from the
Controllers in response to <span>NextWork</span> requests. <span>Heartbeat</span>
and <span>SetResults</span> requests originate from executor processes during and
after task execution. The HSC interacts with the task store to update the task
status on the kind of request it receives.
</p>
</div>
<div>
@ -165,8 +268,16 @@
</p>
</div>
<div>
<p> ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects can be Entities or Associations (<span>assoc</span>), each of which can have user-defined attributes. Associations are used to represent relationships between entities. Edgestore supports indexing only on attributes of associations. </p>
<p> Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF association stores scheduling information, such as the next scheduled timestamp at which the Store Consumer should poll a given task (either for the first time or for a retry). The ATF entity stores all task related information that is used to track the task state and payload for task execution. We query on associations from the Store Consumer in a pull model to pick up tasks ready for execution. </p>
<p> ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects
can be Entities or Associations (<span>assoc</span>), each of which can have
user-defined attributes. Associations are used to represent relationships between
entities. Edgestore supports indexing only on attributes of associations. </p>
<p> Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF
association stores scheduling information, such as the next scheduled timestamp at
which the Store Consumer should poll a given task (either for the first time or for
a retry). The ATF entity stores all task related information that is used to track
the task state and payload for task execution. We query on associations from the
Store Consumer in a pull model to pick up tasks ready for execution. </p>
</div>
<div>
<p id="lifecycle-of-a-task">
@ -175,185 +286,212 @@
</div>
<div>
<ol>
<li>Client performs a <span>Schedule</span> RPC call to <b>Frontend</b> with task information, including execution time.&#160; </li>
<li>Frontend creates Edgestore <span>entity</span> and <span>assoc</span> for the task.&#160; </li>
<li>When it is time to process the task, <b>Store Consumer</b> pulls the task from <b>Edgestore</b> and pushes it to a related <b>SQS</b> queue.&#160; </li>
<li>
<b>Executor</b> makes <span>NextWork</span> RPC call to <b>Controller</b>, which pulls tasks from the <b>SQS</b> queue, makes a <span>ClaimTask</span> RPC to the HSC and then returns the task to the <b>Executor</b>.&#160;
<li>Client performs a <span>Schedule</span> RPC call to <b>Frontend</b> with task
information, including execution time.&#160;
</li>
<li>Frontend creates Edgestore <span>entity</span> and <span>assoc</span> for the
task.&#160;
</li>
<li>When it is time to process the task, <b>Store Consumer</b> pulls the task from
<b>Edgestore</b> and pushes it to a related <b>SQS</b> queue.&#160;
</li>
<li>
<b>Executor</b> invokes the callback for the task. While processing, <b>Executor</b> performs <span>Heartbeat</span> RPC calls to <b>Heartbeat and Status Controller (HSC)</b>. Once processing is done, <b>Executor</b> performs <span>TaskStatus</span> RPC call to <b>HSC</b>.&#160;
<b>Executor</b> makes <span>NextWork</span> RPC call to <b>Controller</b>, which
pulls tasks from the <b>SQS</b> queue, makes a <span>ClaimTask</span> RPC to the
HSC and then returns the task to the <b>Executor</b>.&#160;
</li>
<li>
<b>Executor</b> invokes the callback for the task. While processing,
<b>Executor</b> performs <span>Heartbeat</span> RPC calls to <b>Heartbeat and
Status Controller (HSC)</b>. Once processing is done, <b>Executor</b> performs
<span>TaskStatus</span> RPC call to <b>HSC</b>.&#160;
</li>
<li>Upon getting <span>Heartbeat</span> and <span>TaskStatus</span> RPC calls, <b>HSC</b>
updates the <b>Edgestore</b> entity and <span>assoc</span>.
</li>
<li>Upon getting <span>Heartbeat</span> and <span>TaskStatus</span> RPC calls, <b>HSC</b> updates the <b>Edgestore</b> entity and <span>assoc</span>. </li>
</ol>
<p> Every state update in the lifecycle of a task is accompanied by an update to the next trigger timestamp in the <span>assoc</span>. This ensures that the Store Consumer pulls the task again if there is no change in state of the task within the next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee by ensuring that no task is dropped. </p>
<p> Following are the task entity and association states in ATF and their corresponding timestamp updates: </p>
<p> Every state update in the lifecycle of a task is accompanied by an update to the
next trigger timestamp in the <span>assoc</span>. This ensures that the Store
Consumer pulls the task again if there is no change in state of the task within the
next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee
by ensuring that no task is dropped. </p>
<p> Following are the task entity and association states in ATF and their corresponding
timestamp updates: </p>
<table>
<tbody>
<tr>
<td>
<p>
<b>Entity status</b>
</p>
</td>
<td>
<p>
<b>Assoc status</b>
</p>
</td>
<td>
<p>
<b>next trigger timestamp in Assoc</b>
</p>
</td>
<td>
<p>
<b>Comment</b>
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>new</span>
</p>
</td>
<td>
<p>
<span>new</span>
</p>
</td>
<td>
<p>
<span>scheduled_timestamp</span> of the task
</p>
</td>
<td>
<p> Pick up new tasks that are ready.&#160; </p>
</td>
</tr>
<tr>
<td>
<p>
<span>enqueued</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>enqueued_timestamp</span> + <span>enqueue_timeout</span>
</p>
</td>
<td>
<p> Re-enqueue task if it has been in <span>enqueued</span> state for too long. This can happen if the queue loses data or the controller goes down after polling the queue and before the task is claimed. </p>
</td>
</tr>
<tr>
<td>
<p>
<span>claimed</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>claimed_timestamp</span> + <span>claim_timeout</span>
</p>
</td>
<td>
<p> Re-enqueue if task is claimed but never transfered to <span>processing</span>. This can happen if Controller is down after claiming a task. Task status is changed to <span>enqueued</span> after re-enqueue. </p>
</td>
</tr>
<tr>
<td>
<p>
<span>processing</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>heartbeat_timestamp</span> + <span>heartbeat_timeout</span>`
</p>
</td>
<td>
<p> Re-enqueue if task hasn’t sent <span>heartbeat</span> for too long. This can happen if Executor is down. Task status is changed to <span>enqueued</span> after re-enqueue.&#160; </p>
</td>
</tr>
<tr>
<td>
<p>
<span>retriable failure</span>
</p>
</td>
<td>
<p> started </p>
</td>
<td>
<p> compute <span>next_timestamp</span> according to backoff logic </p>
</td>
<td>
<p> Exponential backoff for tasks with retriable failure.&#160; </p>
</td>
</tr>
<tr>
<td>
<p>
<span>success</span>
</p>
</td>
<td>
<p>
<span>completed</span>
</p>
</td>
<td>
<p> N/A </p>
</td>
<td>
</td>
</tr>
<tr>
<td>
<p>
<span>fatal_failure</span>
</p>
</td>
<td>
<p>
<span>completed</span>
</p>
</td>
<td>
<p> N/A </p>
</td>
<td>
</td>
</tr>
<tr>
<td>
<p>
<b>Entity status</b>
</p>
</td>
<td>
<p>
<b>Assoc status</b>
</p>
</td>
<td>
<p>
<b>next trigger timestamp in Assoc</b>
</p>
</td>
<td>
<p>
<b>Comment</b>
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>new</span>
</p>
</td>
<td>
<p>
<span>new</span>
</p>
</td>
<td>
<p>
<span>scheduled_timestamp</span> of the task
</p>
</td>
<td>
<p> Pick up new tasks that are ready.&#160; </p>
</td>
</tr>
<tr>
<td>
<p>
<span>enqueued</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>enqueued_timestamp</span> + <span>enqueue_timeout</span>
</p>
</td>
<td>
<p> Re-enqueue task if it has been in <span>enqueued</span> state for too
long. This can happen if the queue loses data or the controller goes
down after polling the queue and before the task is claimed. </p>
</td>
</tr>
<tr>
<td>
<p>
<span>claimed</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>claimed_timestamp</span> + <span>claim_timeout</span>
</p>
</td>
<td>
<p> Re-enqueue if task is claimed but never transfered to
<span>processing</span>. This can happen if Controller is down after
claiming a task. Task status is changed to <span>enqueued</span> after
re-enqueue. </p>
</td>
</tr>
<tr>
<td>
<p>
<span>processing</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>heartbeat_timestamp</span> + <span>heartbeat_timeout</span>`
</p>
</td>
<td>
<p> Re-enqueue if task hasn’t sent <span>heartbeat</span> for too long. This
can happen if Executor is down. Task status is changed to
<span>enqueued</span> after re-enqueue.&#160; </p>
</td>
</tr>
<tr>
<td>
<p>
<span>retriable failure</span>
</p>
</td>
<td>
<p> started </p>
</td>
<td>
<p> compute <span>next_timestamp</span> according to backoff logic </p>
</td>
<td>
<p> Exponential backoff for tasks with retriable failure.&#160; </p>
</td>
</tr>
<tr>
<td>
<p>
<span>success</span>
</p>
</td>
<td>
<p>
<span>completed</span>
</p>
</td>
<td>
<p> N/A </p>
</td>
<td>
</td>
</tr>
<tr>
<td>
<p>
<span>fatal_failure</span>
</p>
</td>
<td>
<p>
<span>completed</span>
</p>
</td>
<td>
<p> N/A </p>
</td>
<td>
</td>
</tr>
</tbody>
</table>
<p> The store consumer polls for tasks based on the following query: </p>
<p>
<span>assoc_status= &amp;&amp; next_timestamp&lt;=time.now()<br /></span>
<span>assoc_status= &amp;&amp; next_timestamp&lt;=time.now()<br/></span>
</p>
<p> Below is the state machine that defines task state transitions:&#160;<br />
<p> Below is the state machine that defines task state transitions:&#160;<br/>
</p>
</div>
<div>
<figure>
<img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x225px-2.png" aria-hidden="false" alt="Task State Transitions [Fig 2]" height="450" width="1440" />
<img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x225px-2.png"
aria-hidden="false" alt="Task State Transitions [Fig 2]" height="450"
width="1440"/>
</figure>
</div>
<div>
@ -363,19 +501,51 @@
</div>
<div>
<p>
<b>At-least-once task execution<br /></b> At-least-once execution is guaranteed in ATF by retrying a task until it completes execution (which is signaled by a <span>Success</span> or a <span>FatalFailure</span> state). All ATF system errors are implicitly considered retriable failures, and lambda owners have an option of marking tasks with a <span>RetriableFailure</span> state. Tasks might be dropped from the ATF execution pipeline in different parts of the system through transient RPC failures and failures on dependencies like Edgestore or SQS. These transient failures at different parts of the system do not affect the at-least-once guarantee, though, because of the system of timeouts and re-polling from Store Consumer.
<b>At-least-once task execution<br/></b> At-least-once execution is guaranteed in
ATF by retrying a task until it completes execution (which is signaled by a <span>Success</span>
or a <span>FatalFailure</span> state). All ATF system errors are implicitly
considered retriable failures, and lambda owners have an option of marking tasks
with a <span>RetriableFailure</span> state. Tasks might be dropped from the ATF
execution pipeline in different parts of the system through transient RPC failures
and failures on dependencies like Edgestore or SQS. These transient failures at
different parts of the system do not affect the at-least-once guarantee, though,
because of the system of timeouts and re-polling from Store Consumer.
</p>
<p>
<b>No concurrent task execution<br /></b> Concurrent task execution is avoided through a combination of two methods in ATF. First, tasks are explicitly claimed through an exclusive task state (<span>Claimed</span>) before starting execution. Once the task execution is complete, the task status is updated to one of <span>Success</span>, <span>FatalFailure</span> or <span>RetriableFailure</span>. A task can be claimed only if its existing task state is <span>Enqueued</span> (retried tasks go to the <span>Enqueued</span> state as well once they are re-pushed onto SQS).
<b>No concurrent task execution<br/></b> Concurrent task execution is avoided
through a combination of two methods in ATF. First, tasks are explicitly claimed
through an exclusive task state (<span>Claimed</span>) before starting execution.
Once the task execution is complete, the task status is updated to one of <span>Success</span>,
<span>FatalFailure</span> or <span>RetriableFailure</span>. A task can be claimed
only if its existing task state is <span>Enqueued</span> (retried tasks go to the
<span>Enqueued</span> state as well once they are re-pushed onto SQS).
</p>
<p> However, there might be situations where once a long running task starts execution, its heartbeats might fail repeatedly yet the task execution continues. ATF would retry this task by polling it from the store consumer because the heartbeat timeouts would’ve expired. This task can then be claimed by another worker and lead to concurrent execution.&#160;<br />
<p> However, there might be situations where once a long running task starts execution,
its heartbeats might fail repeatedly yet the task execution continues. ATF would
retry this task by polling it from the store consumer because the heartbeat timeouts
would’ve expired. This task can then be claimed by another worker and lead to
concurrent execution.&#160;<br/>
</p>
<p> To avoid this situation, there is a termination logic in the Executor processes whereby an Executor process terminates itself as soon as three consecutive heartbeat calls fail. Each heartbeat timeout is large enough to eclipse three consecutive heartbeat failures. This ensures that the Store Consumer cannot pull such tasks before the termination logic ends them—the second method that helps achieve this guarantee. </p>
<p> To avoid this situation, there is a termination logic in the Executor processes
whereby an Executor process terminates itself as soon as three consecutive heartbeat
calls fail. Each heartbeat timeout is large enough to eclipse three consecutive
heartbeat failures. This ensures that the Store Consumer cannot pull such tasks
before the termination logic ends them—the second method that helps achieve this
guarantee. </p>
<p>
<b>Isolation<br /></b> Isolation of lambdas is achieved through dedicated worker clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition, isolation across different priorities within the same lambda is likewise achieved through dedicated queues and scheduling bandwidth.
<b>Isolation<br/></b> Isolation of lambdas is achieved through dedicated worker
clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition,
isolation across different priorities within the same lambda is likewise achieved
through dedicated queues and scheduling bandwidth.
</p>
<p>
<b>Delivery latency<br /></b> ATF use cases do not require ultra-low task delivery latencies. Task delivery latencies on the order of a couple of seconds are acceptable. Tasks ready for execution are periodically polled by the Store Consumer and this period of polling largely controls the task delivery latency. Using this as a tuning lever, ATF can achieve different delivery latencies as required. Increasing poll frequency reduces task delivery latency and vice versa. Currently, we have calibrated ATF to poll for ready tasks once every two seconds.
<b>Delivery latency<br/></b> ATF use cases do not require ultra-low task delivery
latencies. Task delivery latencies on the order of a couple of seconds are
acceptable. Tasks ready for execution are periodically polled by the Store Consumer
and this period of polling largely controls the task delivery latency. Using this as
a tuning lever, ATF can achieve different delivery latencies as required. Increasing
poll frequency reduces task delivery latency and vice versa. Currently, we have
calibrated ATF to poll for ready tasks once every two seconds.
</p>
</div>
<div>
@ -383,24 +553,52 @@
<h2> Ownership model </h2>
</p>
</div>
<p> ATF is designed to be a self-serve framework for developers at Dropbox. The design is very intentional in driving an ownership model where lambda owners own all aspects of their lambdas’ operations. To promote this, all lambda worker clusters are owned by the lambda owners. They have full control over operations on these clusters, including code deployments and capacity management. Each executor process is bound to one lambda. Owners have the option of deploying multiple lambdas on their worker clusters simply by spawning new executor processes on their hosts. </p>
<p> ATF is designed to be a self-serve framework for developers at Dropbox. The design is
very intentional in driving an ownership model where lambda owners own all aspects of
their lambdas’ operations. To promote this, all lambda worker clusters are owned by the
lambda owners. They have full control over operations on these clusters, including code
deployments and capacity management. Each executor process is bound to one lambda.
Owners have the option of deploying multiple lambdas on their worker clusters simply by
spawning new executor processes on their hosts. </p>
<div>
<p id="-extending-atf">
<h2> Extending ATF </h2>
</p>
</div>
<div>
<p> As described above, ATF provides an infrastructural building block for scheduling asynchronous tasks. With this foundation established, ATF can be extended to support more generic use cases and provide more features as a framework. Following are some examples of what could be built as an extension to ATF.&#160; </p>
<p> As described above, ATF provides an infrastructural building block for scheduling
asynchronous tasks. With this foundation established, ATF can be extended to support
more generic use cases and provide more features as a framework. Following are some
examples of what could be built as an extension to ATF.&#160; </p>
<p>
<b>Periodic task execution<br /></b> Currently, ATF is a system for one-time task scheduling. Building support for periodic task execution as an extension to this framework would be useful in unlocking new capabilities for our clients.
<b>Periodic task execution<br/></b> Currently, ATF is a system for one-time task
scheduling. Building support for periodic task execution as an extension to this
framework would be useful in unlocking new capabilities for our clients.
</p>
<p>
<b>Better support for task chaining<br /></b> Currently, it is possible to chain tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF during its execution. Although it is possible to do this in the current ATF setup, visibility and control on this chaining is absent at the framework level. Another natural extension here would be to better support task chaining through framework-level visibility and control, to make this use case a first class concept in the ATF model.
<b>Better support for task chaining<br/></b> Currently, it is possible to chain
tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF
during its execution. Although it is possible to do this in the current ATF setup,
visibility and control on this chaining is absent at the framework level. Another
natural extension here would be to better support task chaining through
framework-level visibility and control, to make this use case a first class concept
in the ATF model.
</p>
<p>
<b>Dead letter queues for misbehaving tasks<br /></b> One common source of maintenance overhead we observe on ATF is that some tasks get stuck in infinite retry loops due to occasional bugs in lambda logic. This requires manual intervention from the ATF framework owners in some cases where there are a large number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in the system. Typical manual actions in response to such a situation include pausing execution of the lambdas with misbehaving tasks, or dropping them outright.
<b>Dead letter queues for misbehaving tasks<br/></b> One common source of
maintenance overhead we observe on ATF is that some tasks get stuck in infinite
retry loops due to occasional bugs in lambda logic. This requires manual
intervention from the ATF framework owners in some cases where there are a large
number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in
the system. Typical manual actions in response to such a situation include pausing
execution of the lambdas with misbehaving tasks, or dropping them outright.
</p>
<p> One way to reduce this operational overhead and provide an easy interface for lambda owners to recover from such incidents would be to create dead letter queues filled with such misbehaving tasks. The ATF framework could impose a maximum number of retries before tasks are pushed onto the dead letter queue. We could create and expose tools that make it easy to reschedule tasks from the dead letter queue back into the ATF system, once the associated lambda bugs are fixed.<br />
<p> One way to reduce this operational overhead and provide an easy interface for lambda
owners to recover from such incidents would be to create dead letter queues filled
with such misbehaving tasks. The ATF framework could impose a maximum number of
retries before tasks are pushed onto the dead letter queue. We could create and
expose tools that make it easy to reschedule tasks from the dead letter queue back
into the ATF system, once the associated lambda bugs are fixed.<br/>
</p>
</div>
<div>
@ -408,6 +606,9 @@
<h2> Conclusion </h2>
</p>
</div>
<p> We hope this post helps engineers elsewhere to develop better async task frameworks of their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong, Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.<br /> &#160; </p>
<p> We hope this post helps engineers elsewhere to develop better async task frameworks of
their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak
Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong,
Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.<br/> &#160; </p>
</div>
</div>

2043
app/src/main/assets/readability/test/test-pages/dropbox-blog/source.html
View File

File diff suppressed because it is too large Load Diff

196
app/src/main/assets/readability/test/test-pages/ebb-org/expected.html
View File

@ -1,26 +1,190 @@
<div id="readability-page-1" class="page">
<div id="contentWithSidebar">
<p> Tuesday 15 October 2019 by Bradley M. Kuhn </p>
<p> The last 33 days have been unprecedentedly difficult for the software freedom community and for me personally. Folks have been emailing, phoning, texting, tagging me on social media (— the last of which has been funny, because all my social media accounts are placeholder accounts). But, just about everyone has urged me to comment on the serious issues that the software freedom community now faces. Until now, I have stayed silent regarding all these current topics: from Richard M. Stallman (RMS)'s public statements, to <a href="https://www.fsf.org/news/richard-m-stallman-resigns">his resignation from the Free Software Foundation (FSF)</a>, to the Epstein scandal and its connection to MIT. I've also avoided generally commenting on software freedom organizational governance during this period. I did this for good reason, which is explained below. However, in this blog post, I now share my primary comments on the matters that seem to currently be of the utmost attention of the Open Source and Free Software communities. </p>
<p> I have been silent the last month because, until two days ago, I was an at-large member of <a href="https://www.fsf.org/about/staff-and-board">FSF's Board of Directors</a>, and a <a href="https://static.fsf.org/nosvn/fsf-amended-bylaws-current.pdf">Voting Member</a> of the FSF. As a member of FSF's two leadership bodies, I was abiding by a reasonable request from the FSF management and my duty to the organization. Specifically, the FSF asked that all communication during the crisis <a href="https://www.fsf.org/news/richard-m-stallman-resigns">come</a> <a href="https://www.fsf.org/news/fsf-and-gnu">directly</a> from FSF officers and not from at-large directors and/or Voting Members. Furthermore, the FSF management asked all Directors and Voting Members to remain silent on this entire matter — even on issues only tangentially related to the current situation, and even when speaking in our own capacity (e.g., on our own blogs like this one). The FSF is an important organization, and I take any request from the FSF seriously — so I abided fully with their request. </p>
<p> The situation was further complicated because folks at my employer, Software Freedom Conservancy (where I also serve on the <a href="https://sfconservancy.org/about/board/#bkuhn">Board of Directors</a>) had strong opinions about this matter as well. Fortunately, the FSF and Conservancy both had already created clear protocols for what I should do if ever there was a disagreement or divergence of views between Conservancy and FSF. I therefore was recused fully from the planning, drafting, and timing of Conservancy's statement on this matter. I thank my colleagues at the Conservancy for working so carefully to keep me entirely outside the loop on their statement and to diligently assure that it was straight-forward for me to manage any potential organizational disagreements. I also thank those at the FSF who outlined clear protocols (ahead of time, back in March 2019) in case a situation like this ever came up. I also know my colleagues at Conservancy care deeply, as I do, about the health and welfare of the FSF and its mission of fighting for universal software freedom for all. None of us want, nor have, any substantive disagreement over software freedom issues. </p>
<p> I take very seriously my duty to the various organizations where I have (or have had) affiliations. More generally, I champion non-profit organizational transparency. Unfortunately, the current crisis left me in a quandary between the overarching goal of community transparency and abiding by FSF management's directives. Now that I've left the FSF Board of Directors, FSF's Voting Membership, and all my FSF volunteer roles (which ends my 22-year uninterrupted affiliation with the FSF), I can now comment on the substantive issues that face not just the FSF, but the Free Software community as a whole, while continuing to adhere to my past duty of acting in FSF's best interest. In other words, my affiliation with the FSF has come to an end for many good and useful reasons. The end to this affiliation allows me to speak directly about the core issues at the heart of the community's current crisis. </p>
<p> Firstly, all these events — from RMS' public comments on the MIT mailing list, to RMS' resignation from the FSF to RMS' discussions about the next steps for the GNU project — <em>seem</em> to many to have happened ridiculously quickly. But it wasn't actually fast at all. In fact, these events were culmination of issues that were slowly growing in concern to many people, including me. </p>
<p> For the last two years, I had been a loud internal voice in the FSF leadership regarding RMS' Free-Software-unrelated public statements; I felt strongly that it was in the best interest of the FSF to actively seek to limit such statements, and that it was my duty to FSF to speak out about this within the organization. Those who only learned of this story in the last month (understandably) believed <a href="https://medium.com/@selamjie/remove-richard-stallman-fec6ec210794">Selam G.'s Medium post</a> raised an entirely new issue. <a href="https://web.archive.org/web/20161107050933/https://www.stallman.org/archives/2016-jul-oct.html#31_October_2016_(Down&apos;s_syndrome)">In</a> <a href="https://web.archive.org/web/20170202025227/https://www.stallman.org/archives/2016-nov-feb.html#14_December_2016_(Campaign_of_bull-headed_prudery)">fact</a>, <a href="https://web.archive.org/web/20170224174306/https://www.stallman.org/archives/2016-nov-feb.html#23_February_2017_(A_violent_sex_offender)">RMS'</a> <a href="https://web.archive.org/web/20170612074722/http://stallman.org/archives/2017-mar-jun.html#26_May_2017_(Prudish_ignorantism)">views</a> <a href="https://web.archive.org/web/20170616044924/https://www.stallman.org/archives/2017-mar-jun.html#13_June_2017_(Sex_offender_registry)">and</a> <a href="https://web.archive.org/web/20171020041022/http://stallman.org/archives/2017-jul-oct.html#10_October_2017_(Laws_against_having_sex_with_an_animal)">statements</a> <a href="https://web.archive.org/web/20180131020215/https://stallman.org/archives/2017-jul-oct.html#29_October_2017_(Pestering_women)">posted</a> <a href="https://web.archive.org/web/20180104112431/https://www.stallman.org/archives/2017-nov-feb.html#27_November_2017_(Roy_Moore&apos;s_relationships)">on</a> <a href="https://web.archive.org/web/20180509120046/https://stallman.org/archives/2018-mar-jun.html#30_April_2018_(UN_peacekeepers_in_South_Sudan)">stallman.org</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#17_July_2018_(The_bullshitter&apos;s_flirting)">about</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#21_August_2018_(Age_and_attraction)">sexual</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">morality</a> <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">escalated</a> <a href="https://web.archive.org/web/20181113161736/https://www.stallman.org/archives/2018-sep-dec.html#6_November_2018_(Sex_according_to_porn)">for</a> <a href="https://web.archive.org/web/20190325024048/https://stallman.org/archives/2019-jan-apr.html#14_February_2019_(Respecting_peoples_right_to_say_no)">the</a> <a href="https://www.stallman.org/archives/2019-may-aug.html#11_June_2019_(Stretching_meaning_of_terms)">worse</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#12_June_2019_(Declining_sex_rates)">over</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#30_July_2019_(Al_Franken)">the</a> <a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">last</a> <a href="https://web.archive.org/web/20191011023557/https://stallman.org/archives/2019-jul-oct.html#21_September_2019_(Sex_workers)">few</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">years</a>. When the escalation started, I still considered RMS both a friend and colleague, and I attempted to argue with him at length to convince him that some of his positions were harmful to sexual assault survivors and those who are sex trafficked, and to the people who devote their lives in service to such individuals. More importantly to the FSF, I attempted to persuade RMS that launching a controversial campaign on sexual behavior and morality was counter to his and FSF's mission to advance software freedom, and told RMS that my duty as an FSF Director was to assure the best outcome for the FSF, which <acronym title="in my opinion">IMO</acronym> didn't include having a leader who made such statements. Not only is human sexual behavior not a topic on which RMS has adequate academic expertise, but also his positions appear to ignore significant research and widely available information on the subject. Many of his comments, while occasionally politically intriguing, lack empathy for people who experienced trauma. </p>
<p> IMO, this is not and has never been a Free Speech issue. I do believe freedom of speech links directly to software freedom: indeed, I see the freedom to publish software under Free licenses as almost a corollary to the freedom of speech. However, we do not need to follow leadership from those whose views we fundamentally disagree. Moreover, organizations need not and should not elevate spokespeople and leaders who speak regularly on unrelated issues that organizations find do not advance their mission, and/or that alienate important constituents. I, like many other software freedom leaders, curtail my public comments on issues not related to <acronym title="Free and Open Source Software">FOSS</acronym>. (Indeed, I would not even be commenting on <em>this issue</em> if it had not become a central issue of concern to the software freedom community.) Leaders have power, and they must exercise the power of their words with <a href="https://lwn.net/Articles/770966/">restraint, not with impunity</a>. </p>
<p> RMS has consistently argued that there was a campaign of “prudish intimidation” — seeking to keep him quiet about his views on sexuality. After years of conversing with RMS about how his non-software-freedom views were a distraction, an indulgence, and downright problematic, his general response was to make even more public comments of this nature. The issue is not about RMS' right to say what he believes, nor is it even about whether or not you agree or disagree with RMS' statements. The question is whether an organization should have a designated leader who is on a sustained, public campaign advocating about an unrelated issue that many consider controversial. It really doesn't matter what your view about the controversial issue is; a leader who refuses to stop talking loudly about unrelated issues eventually creates an untenable distraction from the radical activism you're actively trying to advance. The message of universal software freedom is a radical cause; it's basically impossible for one individual to effectively push forward two unrelated controversial agendas at once. In short, the radical message of software freedom became overshadowed by RMS' radical views about sexual morality. </p>
<p> And here is where I say the thing that may infuriate many but it's what I believe: I think RMS took a useful step by resigning some of his leadership roles at the FSF. I thank RMS for taking that step, and I wish the FSF Directors well in their efforts to assure that the FSF becomes a welcoming organization to all who care about universal software freedom. The <a href="https://www.fsf.org/about/">FSF's mission</a> is essential to our technological future, and we should all support that mission. I care deeply about that mission myself and have worked and will continue to work in our community in the best interest of the mission. </p>
<p> I'm admittedly struggling to find a way to work again with RMS, given his views on sexual morality and his behaviors stemming from those views. I explicitly do not agree with <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">this “(re-)definition” of sexual assault</a>. Furthermore, I believe uninformed statements about sexual assault are irresponsible and cause harm to victims. #MeToo is <strong><a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">not a “frenzy”</a></strong>; it is a global movement by individuals who have been harmed seeking to hold both bad actors <em>and</em> society-at-large accountable for ignoring systemic wrongs. Nevertheless, I still am proud of the <a href="https://www.gnu.org/philosophy/freedom-or-power.en.html">essay that I co-wrote with RMS</a> and still find <a href="https://www.gnu.org/gnu/manifesto.en.html">many</a> <a href="https://www.gnu.org/philosophy/free-sw.html">of</a> <a href="https://www.gnu.org/philosophy/why-free.html">RMS'</a> <a href="https://www.gnu.org/philosophy/pragmatic.html">other</a> <a href="https://www.gnu.org/philosophy/microsoft-old.html">essays</a> <a href="https://www.gnu.org/philosophy/gpl-american-way.html">compelling</a>, <a href="https://www.gnu.org/licenses/why-not-lgpl.html">important</a>, <a href="https://www.gnu.org/philosophy/stallman-kth.en.html">and</a> <a href="https://www.gnu.org/philosophy/who-does-that-server-really-serve.en.html">relevant</a>. </p>
<p> I want the FSF to succeed in its mission and enter a new era of accomplishments. I've spent the last 22 years, without a break, dedicating substantial time, effort, care and loyalty to the various FSF roles that I've had: including employee, volunteer, at-large Director, and Voting Member. Even though my duties to the FSF are done, and my relationship with the FSF is no longer formal, I still think the FSF is a valuable institution worth helping and saving, specifically because the FSF was founded for a mission that I deeply support. And we should also realize that RMS — a human being (who is flawed like the rest of us) — invented that mission. </p>
<p> As culture change becomes more rapid, I hope we can find reasonable nuance and moderation on our complex analysis about people and their disparate views, while we also hold individuals fully accountable for their actions. That's the difficulty we face in the post-post-modern culture of the early twenty-first century. Most importantly, I believe we must find a way to stand firm for software freedom while also making a safe environment for victims of sexual assault, sexual abuse, gaslighting, and other deplorable actions. </p>
<p> The last 33 days have been unprecedentedly difficult for the software freedom community
and for me personally. Folks have been emailing, phoning, texting, tagging me on social
media (— the last of which has been funny, because all my social media accounts are
placeholder accounts). But, just about everyone has urged me to comment on the serious
issues that the software freedom community now faces. Until now, I have stayed silent
regarding all these current topics: from Richard M. Stallman (RMS)'s public statements,
to <a href="https://www.fsf.org/news/richard-m-stallman-resigns">his resignation from
the Free Software Foundation (FSF)</a>, to the Epstein scandal and its connection to
MIT. I've also avoided generally commenting on software freedom organizational
governance during this period. I did this for good reason, which is explained below.
However, in this blog post, I now share my primary comments on the matters that seem to
currently be of the utmost attention of the Open Source and Free Software communities.
</p>
<p> I have been silent the last month because, until two days ago, I was an at-large member
of <a href="https://www.fsf.org/about/staff-and-board">FSF's Board of Directors</a>, and
a <a href="https://static.fsf.org/nosvn/fsf-amended-bylaws-current.pdf">Voting
Member</a> of the FSF. As a member of FSF's two leadership bodies, I was abiding by
a reasonable request from the FSF management and my duty to the organization.
Specifically, the FSF asked that all communication during the crisis <a
href="https://www.fsf.org/news/richard-m-stallman-resigns">come</a> <a
href="https://www.fsf.org/news/fsf-and-gnu">directly</a> from FSF officers and
not from at-large directors and/or Voting Members. Furthermore, the FSF management asked
all Directors and Voting Members to remain silent on this entire matter — even on issues
only tangentially related to the current situation, and even when speaking in our own
capacity (e.g., on our own blogs like this one). The FSF is an important organization,
and I take any request from the FSF seriously — so I abided fully with their request.
</p>
<p> The situation was further complicated because folks at my employer, Software Freedom
Conservancy (where I also serve on the <a
href="https://sfconservancy.org/about/board/#bkuhn">Board of Directors</a>) had
strong opinions about this matter as well. Fortunately, the FSF and Conservancy both had
already created clear protocols for what I should do if ever there was a disagreement or
divergence of views between Conservancy and FSF. I therefore was recused fully from the
planning, drafting, and timing of Conservancy's statement on this matter. I thank my
colleagues at the Conservancy for working so carefully to keep me entirely outside the
loop on their statement and to diligently assure that it was straight-forward for me to
manage any potential organizational disagreements. I also thank those at the FSF who
outlined clear protocols (ahead of time, back in March 2019) in case a situation like
this ever came up. I also know my colleagues at Conservancy care deeply, as I do, about
the health and welfare of the FSF and its mission of fighting for universal software
freedom for all. None of us want, nor have, any substantive disagreement over software
freedom issues. </p>
<p> I take very seriously my duty to the various organizations where I have (or have had)
affiliations. More generally, I champion non-profit organizational transparency.
Unfortunately, the current crisis left me in a quandary between the overarching goal of
community transparency and abiding by FSF management's directives. Now that I've left
the FSF Board of Directors, FSF's Voting Membership, and all my FSF volunteer roles
(which ends my 22-year uninterrupted affiliation with the FSF), I can now comment on the
substantive issues that face not just the FSF, but the Free Software community as a
whole, while continuing to adhere to my past duty of acting in FSF's best interest. In
other words, my affiliation with the FSF has come to an end for many good and useful
reasons. The end to this affiliation allows me to speak directly about the core issues
at the heart of the community's current crisis. </p>
<p> Firstly, all these events — from RMS' public comments on the MIT mailing list, to RMS'
resignation from the FSF to RMS' discussions about the next steps for the GNU project —
<em>seem</em> to many to have happened ridiculously quickly. But it wasn't actually fast
at all. In fact, these events were culmination of issues that were slowly growing in
concern to many people, including me. </p>
<p> For the last two years, I had been a loud internal voice in the FSF leadership regarding
RMS' Free-Software-unrelated public statements; I felt strongly that it was in the best
interest of the FSF to actively seek to limit such statements, and that it was my duty
to FSF to speak out about this within the organization. Those who only learned of this
story in the last month (understandably) believed <a
href="https://medium.com/@selamjie/remove-richard-stallman-fec6ec210794">Selam
G.'s Medium post</a> raised an entirely new issue. <a
href="https://web.archive.org/web/20161107050933/https://www.stallman.org/archives/2016-jul-oct.html#31_October_2016_(Down&apos;s_syndrome)">In</a>
<a href="https://web.archive.org/web/20170202025227/https://www.stallman.org/archives/2016-nov-feb.html#14_December_2016_(Campaign_of_bull-headed_prudery)">fact</a>,
<a href="https://web.archive.org/web/20170224174306/https://www.stallman.org/archives/2016-nov-feb.html#23_February_2017_(A_violent_sex_offender)">RMS'</a>
<a href="https://web.archive.org/web/20170612074722/http://stallman.org/archives/2017-mar-jun.html#26_May_2017_(Prudish_ignorantism)">views</a>
<a href="https://web.archive.org/web/20170616044924/https://www.stallman.org/archives/2017-mar-jun.html#13_June_2017_(Sex_offender_registry)">and</a>
<a href="https://web.archive.org/web/20171020041022/http://stallman.org/archives/2017-jul-oct.html#10_October_2017_(Laws_against_having_sex_with_an_animal)">statements</a>
<a href="https://web.archive.org/web/20180131020215/https://stallman.org/archives/2017-jul-oct.html#29_October_2017_(Pestering_women)">posted</a>
<a href="https://web.archive.org/web/20180104112431/https://www.stallman.org/archives/2017-nov-feb.html#27_November_2017_(Roy_Moore&apos;s_relationships)">on</a>
<a href="https://web.archive.org/web/20180509120046/https://stallman.org/archives/2018-mar-jun.html#30_April_2018_(UN_peacekeepers_in_South_Sudan)">stallman.org</a>
<a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#17_July_2018_(The_bullshitter&apos;s_flirting)">about</a>
<a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#21_August_2018_(Age_and_attraction)">sexual</a>
<a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">morality</a>
<a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">escalated</a>
<a href="https://web.archive.org/web/20181113161736/https://www.stallman.org/archives/2018-sep-dec.html#6_November_2018_(Sex_according_to_porn)">for</a>
<a href="https://web.archive.org/web/20190325024048/https://stallman.org/archives/2019-jan-apr.html#14_February_2019_(Respecting_peoples_right_to_say_no)">the</a>
<a href="https://www.stallman.org/archives/2019-may-aug.html#11_June_2019_(Stretching_meaning_of_terms)">worse</a>
<a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#12_June_2019_(Declining_sex_rates)">over</a>
<a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#30_July_2019_(Al_Franken)">the</a>
<a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">last</a>
<a href="https://web.archive.org/web/20191011023557/https://stallman.org/archives/2019-jul-oct.html#21_September_2019_(Sex_workers)">few</a>
<a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">years</a>.
When the escalation started, I still considered RMS both a friend and colleague, and I
attempted to argue with him at length to convince him that some of his positions were
harmful to sexual assault survivors and those who are sex trafficked, and to the people
who devote their lives in service to such individuals. More importantly to the FSF, I
attempted to persuade RMS that launching a controversial campaign on sexual behavior and
morality was counter to his and FSF's mission to advance software freedom, and told RMS
that my duty as an FSF Director was to assure the best outcome for the FSF, which
<acronym title="in my opinion">IMO</acronym> didn't include having a leader who made
such statements. Not only is human sexual behavior not a topic on which RMS has adequate
academic expertise, but also his positions appear to ignore significant research and
widely available information on the subject. Many of his comments, while occasionally
politically intriguing, lack empathy for people who experienced trauma. </p>
<p> IMO, this is not and has never been a Free Speech issue. I do believe freedom of speech
links directly to software freedom: indeed, I see the freedom to publish software under
Free licenses as almost a corollary to the freedom of speech. However, we do not need to
follow leadership from those whose views we fundamentally disagree. Moreover,
organizations need not and should not elevate spokespeople and leaders who speak
regularly on unrelated issues that organizations find do not advance their mission,
and/or that alienate important constituents. I, like many other software freedom
leaders, curtail my public comments on issues not related to <acronym
title="Free and Open Source Software">FOSS</acronym>. (Indeed, I would not even
be commenting on <em>this issue</em> if it had not become a central issue of concern to
the software freedom community.) Leaders have power, and they must exercise the power of
their words with <a href="https://lwn.net/Articles/770966/">restraint, not with
impunity</a>. </p>
<p> RMS has consistently argued that there was a campaign of “prudish intimidation” —
seeking to keep him quiet about his views on sexuality. After years of conversing with
RMS about how his non-software-freedom views were a distraction, an indulgence, and
downright problematic, his general response was to make even more public comments of
this nature. The issue is not about RMS' right to say what he believes, nor is it even
about whether or not you agree or disagree with RMS' statements. The question is whether
an organization should have a designated leader who is on a sustained, public campaign
advocating about an unrelated issue that many consider controversial. It really doesn't
matter what your view about the controversial issue is; a leader who refuses to stop
talking loudly about unrelated issues eventually creates an untenable distraction from
the radical activism you're actively trying to advance. The message of universal
software freedom is a radical cause; it's basically impossible for one individual to
effectively push forward two unrelated controversial agendas at once. In short, the
radical message of software freedom became overshadowed by RMS' radical views about
sexual morality. </p>
<p> And here is where I say the thing that may infuriate many but it's what I believe: I
think RMS took a useful step by resigning some of his leadership roles at the FSF. I
thank RMS for taking that step, and I wish the FSF Directors well in their efforts to
assure that the FSF becomes a welcoming organization to all who care about universal
software freedom. The <a href="https://www.fsf.org/about/">FSF's mission</a> is
essential to our technological future, and we should all support that mission. I care
deeply about that mission myself and have worked and will continue to work in our
community in the best interest of the mission. </p>
<p> I'm admittedly struggling to find a way to work again with RMS, given his views on
sexual morality and his behaviors stemming from those views. I explicitly do not agree
with
<a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">this
“(re-)definition” of sexual assault</a>. Furthermore, I believe uninformed
statements about sexual assault are irresponsible and cause harm to victims. #MeToo is
<strong><a
href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">not
a “frenzy”</a></strong>; it is a global movement by individuals who have been harmed
seeking to hold both bad actors <em>and</em> society-at-large accountable for ignoring
systemic wrongs. Nevertheless, I still am proud of the <a
href="https://www.gnu.org/philosophy/freedom-or-power.en.html">essay that I
co-wrote with RMS</a> and still find <a
href="https://www.gnu.org/gnu/manifesto.en.html">many</a> <a
href="https://www.gnu.org/philosophy/free-sw.html">of</a> <a
href="https://www.gnu.org/philosophy/why-free.html">RMS'</a> <a
href="https://www.gnu.org/philosophy/pragmatic.html">other</a> <a
href="https://www.gnu.org/philosophy/microsoft-old.html">essays</a> <a
href="https://www.gnu.org/philosophy/gpl-american-way.html">compelling</a>, <a
href="https://www.gnu.org/licenses/why-not-lgpl.html">important</a>, <a
href="https://www.gnu.org/philosophy/stallman-kth.en.html">and</a> <a
href="https://www.gnu.org/philosophy/who-does-that-server-really-serve.en.html">relevant</a>.
</p>
<p> I want the FSF to succeed in its mission and enter a new era of accomplishments. I've
spent the last 22 years, without a break, dedicating substantial time, effort, care and
loyalty to the various FSF roles that I've had: including employee, volunteer, at-large
Director, and Voting Member. Even though my duties to the FSF are done, and my
relationship with the FSF is no longer formal, I still think the FSF is a valuable
institution worth helping and saving, specifically because the FSF was founded for a
mission that I deeply support. And we should also realize that RMS — a human being (who
is flawed like the rest of us) — invented that mission. </p>
<p> As culture change becomes more rapid, I hope we can find reasonable nuance and
moderation on our complex analysis about people and their disparate views, while we also
hold individuals fully accountable for their actions. That's the difficulty we face in
the post-post-modern culture of the early twenty-first century. Most importantly, I
believe we must find a way to stand firm for software freedom while also making a safe
environment for victims of sexual assault, sexual abuse, gaslighting, and other
deplorable actions. </p>
<p> Posted on Tuesday 15 October 2019 at 09:11 by Bradley M. Kuhn. </p>
</div>
<p>
<code>#include &lt;std/disclaimer.h&gt;</code><br />
<code>use Standard::Disclaimer;</code><br />
<code>from standard import disclaimer</code><br />
<code>#include &lt;std/disclaimer.h&gt;</code><br/>
<code>use Standard::Disclaimer;</code><br/>
<code>from standard import disclaimer</code><br/>
<code>SELECT full_text FROM standard WHERE type = 'disclaimer';</code>
</p>
<p> Both previously and presently, I have been employed by and/or done work for various organizations that also have views on Free, Libre, and Open Source Software. As should be blatantly obvious, this is my website, not theirs, so please do not assume views and opinions here belong to any such organization. Since I do co-own ebb.org with my wife, it may not be so obvious that these aren't her views and opinions, either. </p>
<p> Both previously and presently, I have been employed by and/or done work for various
organizations that also have views on Free, Libre, and Open Source Software. As should be
blatantly obvious, this is my website, not theirs, so please do not assume views and
opinions here belong to any such organization. Since I do co-own ebb.org with my wife, it
may not be so obvious that these aren't her views and opinions, either. </p>
<p> ebb <sup>℠</sup> is a service mark of Bradley M. Kuhn. </p>
</div>

1144
app/src/main/assets/readability/test/test-pages/ebb-org/source.html
View File

File diff suppressed because it is too large Load Diff

102
app/src/main/assets/readability/test/test-pages/ehow-1/expected.html
View File

@ -3,9 +3,20 @@
<header>
</header>
<div>
<p>Glass cloche terrariums are not only appealing to the eye, but they also preserve a bit of nature in your home and serve as a simple, yet beautiful, piece of art. Closed terrariums are easy to care for, as they retain much of their own moisture and provide a warm environment with a consistent level of humidity. You won’t have to water the terrariums unless you see that the walls are not misting up. Small growing plants that don’t require a lot of light work best such as succulents, ferns, moss, even orchids.</p>
<figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg" alt="Glass cloche terrariums" title="Glass cloche terrariums" data-credit="Lucy Akins " longdesc="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg" /> </figure>
<figcaption class="caption"> Glass cloche terrariums (Lucy Akins) </figcaption>
<p>Glass cloche terrariums are not only appealing to the eye, but they also preserve a
bit of nature in your home and serve as a simple, yet beautiful, piece of art.
Closed terrariums are easy to care for, as they retain much of their own moisture
and provide a warm environment with a consistent level of humidity. You won’t have
to water the terrariums unless you see that the walls are not misting up. Small
growing plants that don’t require a lot of light work best such as succulents,
ferns, moss, even orchids.</p>
<figure><img
src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"
alt="Glass cloche terrariums" title="Glass cloche terrariums"
data-credit="Lucy Akins "
longdesc="http://s3.amazonaws.com/photography.prod.demandstudios.com/16149374-814f-40bc-baf3-ca20f149f0ba.jpg"/>
</figure>
<figcaption class="caption"> Glass cloche terrariums (Lucy Akins)</figcaption>
</div>
<div id="relatedContentUpper" data-module="rcp_top">
<header>
@ -18,7 +29,7 @@
<li>Cloche</li>
<li>Planter saucer, small shallow dish or desired platform</li>
<li>Floral foam oasis</li>
<li>Ruler </li>
<li>Ruler</li>
<li>Spoon</li>
<li>Floral wire pins or paper clips</li>
<li>Small plants (from a florist or nursery)</li>
@ -30,82 +41,113 @@
<div>
<div>
<p><span>Step 1</span></p>
<p>Measure the circumference of your cloche and cut the foam oasis about 3/4 inch (2 cm) smaller. Place the foam oasis into a container full of water and allow to soak until it sinks to the bottom. Dig out a hole on the oasis large enough to fit your plant, being careful not to pierce all the way through to the bottom.</p>
<p>Measure the circumference of your cloche and cut the foam oasis about 3/4 inch (2
cm) smaller. Place the foam oasis into a container full of water and allow to
soak until it sinks to the bottom. Dig out a hole on the oasis large enough to
fit your plant, being careful not to pierce all the way through to the
bottom.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/fc249ef6-4d27-41b4-8c21-15f7a8512b50.jpg" alt="Dig a hole in the oasis." data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Dig a hole in the oasis. (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/fc249ef6-4d27-41b4-8c21-15f7a8512b50.jpg"
alt="Dig a hole in the oasis." data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Dig a hole in the oasis. (Lucy Akins)</figcaption>
</div>
<div>
<div>
<p><span>Step 2</span></p>
<p>Insert your plant into the hole.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/aae11d4d-a4aa-4251-a4d9-41023ebf6d84.jpg" alt="Orchid in foam oasis" data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Orchid in foam oasis (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/aae11d4d-a4aa-4251-a4d9-41023ebf6d84.jpg"
alt="Orchid in foam oasis" data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Orchid in foam oasis (Lucy Akins)</figcaption>
</div>
<div>
<div>
<p><span>Step 3</span></p>
<p>You can add various plants if you wish.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/7afdfa1e-da74-44b5-b89c-ca8123516272.jpg" alt="Various foliage" data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Various foliage (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/7afdfa1e-da74-44b5-b89c-ca8123516272.jpg"
alt="Various foliage" data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Various foliage (Lucy Akins)</figcaption>
</div>
<div>
<div>
<p><span>Step 4</span></p>
<p>Using floral pins, attach enough moss around the oasis to cover it.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/4f6612c0-316a-4c74-bb03-cb4e778f6d72.jpg" alt="Attach moss." data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Attach moss. (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/4f6612c0-316a-4c74-bb03-cb4e778f6d72.jpg"
alt="Attach moss." data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Attach moss. (Lucy Akins)</figcaption>
</div>
<div>
<div>
<p><span>Step 5</span></p>
<p>Gently place the cloche over the oasis. The glass may push some of the moss upward, exposing some of the foam.</p>
<p>Gently place the cloche over the oasis. The glass may push some of the moss
upward, exposing some of the foam.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/eeb1e0b4-e573-40a3-8db1-2c76f0b13b84.jpg" alt="Place cloche over oasis." data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Place cloche over oasis. (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/eeb1e0b4-e573-40a3-8db1-2c76f0b13b84.jpg"
alt="Place cloche over oasis." data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Place cloche over oasis. (Lucy Akins)</figcaption>
</div>
<div>
<div>
<p><span>Step 6</span></p>
<p>Simply pull down the moss with tweezers or insert more moss to fill in the empty spaces.</p>
<p>Simply pull down the moss with tweezers or insert more moss to fill in the empty
spaces.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/812d4649-4152-4363-97c0-f181d02e709a.jpg" alt="Rearrange moss." data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Rearrange moss. (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/812d4649-4152-4363-97c0-f181d02e709a.jpg"
alt="Rearrange moss." data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Rearrange moss. (Lucy Akins)</figcaption>
</div>
<div>
<div>
<p><span>Step 7</span></p>
<p>You can use any platform you wish. In this case, a small saucer was used.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/0cb3988c-9318-47d6-bc9c-c798da1ede72.jpg" alt="Place cloche on a platform to sit on." data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Place cloche on a platform to sit on. (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/default/cme/photography.prod.demandstudios.com/0cb3988c-9318-47d6-bc9c-c798da1ede72.jpg"
alt="Place cloche on a platform to sit on." data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Place cloche on a platform to sit on. (Lucy Akins)
</figcaption>
</div>
<div>
<div>
<p><span>Step 8</span></p>
<p>This particular terrarium rests on a planter saucer and features a small white pumpkin.</p>
<p>This particular terrarium rests on a planter saucer and features a small white
pumpkin.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/e3e18f0b-ab2c-4ffb-9988-a1ea63faef8b.jpg" alt="Cloche placed on a terracotta saucer" data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Cloche placed on a terracotta saucer (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/e3e18f0b-ab2c-4ffb-9988-a1ea63faef8b.jpg"
alt="Cloche placed on a terracotta saucer" data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Cloche placed on a terracotta saucer (Lucy Akins)
</figcaption>
</div>
<div>
<div>
<p><span>Step 9</span></p>
<p>This particular terrarium was placed on a wood slice and a little toy squirrel was placed inside to add a little whimsy.</p>
<p>This particular terrarium was placed on a wood slice and a little toy squirrel
was placed inside to add a little whimsy.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/2cd79f8d-0d16-4573-8861-e47fb74b0638.jpg" alt="Placed on a wooden slice" data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Placed on a wooden slice (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/2cd79f8d-0d16-4573-8861-e47fb74b0638.jpg"
alt="Placed on a wooden slice" data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Placed on a wooden slice (Lucy Akins)</figcaption>
</div>
<div>
<div>
<p><span>Finished Terrarium</span></p>
<p>Displayed alone or in a group, these pretty arrangements allow you to add a little nature to your decor or tablescape.</p>
<p>Displayed alone or in a group, these pretty arrangements allow you to add a
little nature to your decor or tablescape.</p>
</div>
<figure> <img src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/78670312-8636-4c42-a75c-3029f7aa6c73.jpg" alt="Cloche terrarium" data-credit="Lucy Akins" /> </figure>
<figcaption class="caption"> Cloche terrarium (Lucy Akins) </figcaption>
<figure><img
src="http://img-aws.ehowcdn.com/640/cme/photography.prod.demandstudios.com/78670312-8636-4c42-a75c-3029f7aa6c73.jpg"
alt="Cloche terrarium" data-credit="Lucy Akins"/></figure>
<figcaption class="caption"> Cloche terrarium (Lucy Akins)</figcaption>
</div>
<section id="FeaturedTombstone" data-module="rcp_tombstone">
<h2>Featured</h2>

1315
app/src/main/assets/readability/test/test-pages/ehow-1/source.html
View File

File diff suppressed because it is too large Load Diff

168
app/src/main/assets/readability/test/test-pages/ehow-2/expected.html
View File

@ -1,29 +1,45 @@
<div id="readability-page-1" class="page">
<div data-type="AuthorProfile">
<div>
<p><a id="img-follow-tip" href="http://fakehost/contributor/gina_robertsgrey/" target="_top">
<img src="http://img-aws.ehowcdn.com/60x60/cme/cme_public_images/www_demandstudios_com/sitelife.studiod.com/ver1.0/Content/images/store/9/2/d9dd6f61-b183-4893-927f-5b540e45be91.Small.jpg" data-failover="//img-aws.ehowcdn.com/60x60/ehow-cdn-assets/test15/media/images/authors/missing-author-image.png" onerror="var failover = this.getAttribute(&apos;data-failover&apos;);
<p><a id="img-follow-tip" href="http://fakehost/contributor/gina_robertsgrey/"
target="_top">
<img src="http://img-aws.ehowcdn.com/60x60/cme/cme_public_images/www_demandstudios_com/sitelife.studiod.com/ver1.0/Content/images/store/9/2/d9dd6f61-b183-4893-927f-5b540e45be91.Small.jpg"
data-failover="//img-aws.ehowcdn.com/60x60/ehow-cdn-assets/test15/media/images/authors/missing-author-image.png"
onerror="var failover = this.getAttribute(&apos;data-failover&apos;);
if (failover) failover = failover.replace(/^https?:/,&apos;&apos;);
var src = this.src ? this.src.replace(/^https?:/,&apos;&apos;) : &apos;&apos;;
if (src != failover){
this.src = failover;
}" /> </a></p>
}"/> </a></p>
</div>
<p><time datetime="2016-09-14T07:07:00-04:00" itemprop="dateModified">Last updated September 14, 2016</time>
<p>
<time datetime="2016-09-14T07:07:00-04:00" itemprop="dateModified">Last updated
September 14, 2016
</time>
</p>
</div>
<div>
<article data-type="article">
<div>
<div>
<p>Graduation parties are a great way to commemorate the years of hard work teens and college co-eds devote to education. They’re also costly for mom and dad.</p>
<p>The average cost of a graduation party in 2013 was a whopping $1,200, according to Graduationparty.com; $700 of that was allocated for food. However that budget was based on Midwestern statistics, and parties in urban areas like New York City are thought to have a much higher price tag.</p>
<p>Thankfully, there are plenty of creative ways to trim a little grad party fat without sacrificing any of the fun or celebratory spirit.</p>
<p>Graduation parties are a great way to commemorate the years of hard work
teens and college co-eds devote to education. They’re also costly for mom
and dad.</p>
<p>The average cost of a graduation party in 2013 was a whopping $1,200,
according to Graduationparty.com; $700 of that was allocated for food.
However that budget was based on Midwestern statistics, and parties in urban
areas like New York City are thought to have a much higher price tag.</p>
<p>Thankfully, there are plenty of creative ways to trim a little grad party fat
without sacrificing any of the fun or celebratory spirit.</p>
</div>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/2F/86/5547EF62-EAF5-4256-945D-0496F61C862F/5547EF62-EAF5-4256-945D-0496F61C862F.jpg" alt="Graduation" title="Graduation" data-credit="Mike Watson Images/Moodboard/Getty " longdesc="http://s3.amazonaws.com/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/2F/86/5547EF62-EAF5-4256-945D-0496F61C862F/5547EF62-EAF5-4256-945D-0496F61C862F.jpg" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/2F/86/5547EF62-EAF5-4256-945D-0496F61C862F/5547EF62-EAF5-4256-945D-0496F61C862F.jpg"
alt="Graduation" title="Graduation"
data-credit="Mike Watson Images/Moodboard/Getty "
longdesc="http://s3.amazonaws.com/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/2F/86/5547EF62-EAF5-4256-945D-0496F61C862F/5547EF62-EAF5-4256-945D-0496F61C862F.jpg"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> (Mike Watson Images/Moodboard/Getty) </figcaption>
<figcaption class="caption"> (Mike Watson Images/Moodboard/Getty)</figcaption>
</div>
<span>
<span>
@ -31,115 +47,145 @@
<p><span>
<p>Parties hosted at restaurants, clubhouses and country clubs eliminate the need to spend hours cleaning up once party guests have gone home. But that convenience comes with a price tag. A country club may charge as much as $2,000 for room rental and restaurant food and beverage will almost always cost more than food prepped and served at home.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/FE/CB/121569D2-6984-4B2F-83C4-9D2D9A27CBFE/121569D2-6984-4B2F-83C4-9D2D9A27CBFE.jpg" alt="Save money hosting the party at home." data-credit="Thomas Jackson/Digital Vision/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/FE/CB/121569D2-6984-4B2F-83C4-9D2D9A27CBFE/121569D2-6984-4B2F-83C4-9D2D9A27CBFE.jpg"
alt="Save money hosting the party at home."
data-credit="Thomas Jackson/Digital Vision/Getty Images"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> Thomas Jackson/Digital Vision/Getty Images </figcaption>
</div>
</span>
</span>
<span>
<figcaption
class="caption"> Thomas Jackson/Digital Vision/Getty Images </figcaption>
</div>
</span>
</span>
<span>
<span>
<div>
<p><span>
<p>Instead of hiring a DJ, use your iPod or Smartphone to spin the tunes. Both easily hook up to most speakers or mp3 compatible docks to play music from your music library. Or download Pandora, the free online radio app, and play hours of music for free.</p>
<p>Personalize the music with a playlist of the grad’s favorite songs or songs that were big hits during his or her years in school.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/DF/FC/A05B0252-BD73-4BC7-A09A-96F0A504FCDF/A05B0252-BD73-4BC7-A09A-96F0A504FCDF.jpg" alt="Online radio can take the place of a hired DJ." data-credit="Spencer Platt/Getty Images News/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/DF/FC/A05B0252-BD73-4BC7-A09A-96F0A504FCDF/A05B0252-BD73-4BC7-A09A-96F0A504FCDF.jpg"
alt="Online radio can take the place of a hired DJ."
data-credit="Spencer Platt/Getty Images News/Getty Images"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> Spencer Platt/Getty Images News/Getty Images </figcaption>
</div>
</span>
</span>
<span>
<figcaption
class="caption"> Spencer Platt/Getty Images News/Getty Images </figcaption>
</div>
</span>
</span>
<span>
<span>
<div>
<p><span>
<p>Avoid canned drinks, which guests often open, but don't finish. Serve pitchers of tap water with lemon and cucumber slices or sliced strawberries for an interesting and refreshing flavor. Opt for punches and non-alcoholic drinks for high school graduates that allow guests to dole out the exact amount they want to drink.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/EB/DB/8A04CCA7-3255-4225-B59A-C41441F8DBEB/8A04CCA7-3255-4225-B59A-C41441F8DBEB.jpg" alt="Serve drinks in pitchers, not in cans." data-credit="evgenyb/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/EB/DB/8A04CCA7-3255-4225-B59A-C41441F8DBEB/8A04CCA7-3255-4225-B59A-C41441F8DBEB.jpg"
alt="Serve drinks in pitchers, not in cans."
data-credit="evgenyb/iStock/Getty Images"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> evgenyb/iStock/Getty Images </figcaption>
</div>
</div>
</span>
</span>
<span>
</span>
<span>
<span>
<div>
<p><span>
<p>Instead of inviting everyone you – and the graduate – know or ever knew, scale back the guest list. Forgo inviting guests that you or your grad haven't seen for eons. There is no reason to provide provisions for people who are essentially out of your lives. Sticking to a small, but personal, guest list allows more time to mingle with loved ones during the party, too.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/94/10/08035476-0167-4A03-AADC-13A7E7AA1094/08035476-0167-4A03-AADC-13A7E7AA1094.jpg" alt="Limit guests to those close to the graduate." data-credit="Kane Skennar/Photodisc/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/94/10/08035476-0167-4A03-AADC-13A7E7AA1094/08035476-0167-4A03-AADC-13A7E7AA1094.jpg"
alt="Limit guests to those close to the graduate."
data-credit="Kane Skennar/Photodisc/Getty Images"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> Kane Skennar/Photodisc/Getty Images </figcaption>
</div>
<figcaption
class="caption"> Kane Skennar/Photodisc/Getty Images </figcaption>
</div>
</span>
</span>
<span>
</span>
<span>
<span>
<div>
<p><span>
<p>See if your grad and his best friend, girlfriend or close family member would consider hosting a joint party. You can split some of the expenses, especially when the two graduates share mutual friends. You'll also have another parent to bounce ideas off of and to help you stick to your budget when you're tempted to splurge.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/06/49/4AD62696-FC95-4DA2-8351-42740C7B4906/4AD62696-FC95-4DA2-8351-42740C7B4906.jpg" alt="Throw a joint bash for big savings." data-credit="Mike Watson Images/Moodboard/Getty" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/06/49/4AD62696-FC95-4DA2-8351-42740C7B4906/4AD62696-FC95-4DA2-8351-42740C7B4906.jpg"
alt="Throw a joint bash for big savings."
data-credit="Mike Watson Images/Moodboard/Getty"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> Mike Watson Images/Moodboard/Getty </figcaption>
</div>
<figcaption
class="caption"> Mike Watson Images/Moodboard/Getty </figcaption>
</div>
</span>
</span>
<span>
</span>
<span>
<span>
<div>
<p><span>
<p>Skip carving stations of prime rib and jumbo shrimp as appetizers, especially for high school graduation parties. Instead, serve some of the graduate's favorite side dishes that are cost effective, like a big pot of spaghetti with breadsticks. Opt for easy and simple food such as pizza, finger food and mini appetizers. </p>
<p>Avoid pre-packaged foods and pre-made deli platters. These can be quite costly. Instead, make your own cheese and deli platters for less than half the cost of pre-made.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/D0/51/B6AED06C-5E19-4A26-9AAD-0E175F6251D0/B6AED06C-5E19-4A26-9AAD-0E175F6251D0.jpg" alt="Cost effective appetizers are just as satisfying as pre-made deli platters." data-credit="Mark Stout/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/D0/51/B6AED06C-5E19-4A26-9AAD-0E175F6251D0/B6AED06C-5E19-4A26-9AAD-0E175F6251D0.jpg"
alt="Cost effective appetizers are just as satisfying as pre-made deli platters."
data-credit="Mark Stout/iStock/Getty Images"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> Mark Stout/iStock/Getty Images </figcaption>
</div>
</div>
</span>
</span>
<span>
</span>
<span>
<span>
<div>
<p><span>
<p>Instead of an evening dinner party, host a grad lunch or all appetizers party. Brunch and lunch fare or finger food costs less than dinner. Guests also tend to consume less alcohol in the middle of the day, which keeps cost down.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/35/B4/DD5FD05A-B631-4AFE-BC8F-FDACAD1EB435/DD5FD05A-B631-4AFE-BC8F-FDACAD1EB435.jpg" alt="A brunch gathering will cost less than a dinner party." data-credit="Mark Stout/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/35/B4/DD5FD05A-B631-4AFE-BC8F-FDACAD1EB435/DD5FD05A-B631-4AFE-BC8F-FDACAD1EB435.jpg"
alt="A brunch gathering will cost less than a dinner party."
data-credit="Mark Stout/iStock/Getty Images"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> Mark Stout/iStock/Getty Images </figcaption>
</div>
<div id="relatedContentUpper" data-module="rcp_top">
</div>
<div id="relatedContentUpper" data-module="rcp_top">
<header>
<h3>Other People Are Reading</h3>
</header>
</div>
</span>
</span>
<span>
</span>
<span>
<span>
<div>
<p><span>
<p>Decorate your party in the graduate's current school colors or the colors of the school he or she will be headed to next. Décor that is not specifically graduation-themed may cost a bit less, and any leftovers can be re-used for future parties, picnics and events.</p>
</span> </p>
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/A1/FA/2C368B34-8F6A-45F6-9DFC-0B0C4E33FAA1/2C368B34-8F6A-45F6-9DFC-0B0C4E33FAA1.jpg" alt="Theme the party by color without graduation-specific decor." data-credit="jethuynh/iStock/Getty Images" data-pin-ehow-hover="true" data-pin-no-hover="true" />
<figure>
<img src="http://img-aws.ehowcdn.com/640/cme/cme_public_images/www_ehow_com/cdn-write.demandstudios.com/upload/image/A1/FA/2C368B34-8F6A-45F6-9DFC-0B0C4E33FAA1/2C368B34-8F6A-45F6-9DFC-0B0C4E33FAA1.jpg"
alt="Theme the party by color without graduation-specific decor."
data-credit="jethuynh/iStock/Getty Images"
data-pin-ehow-hover="true" data-pin-no-hover="true"/>
</figure>
<figcaption class="caption"> jethuynh/iStock/Getty Images </figcaption>
</div>
</span>
</span>
<h2>
<a target="_blank" href="https://www.google.com/adsense/support/bin/request.py?contact=abg_afc&amp;url=http://ehow.com/&amp;hl=en&amp;client=ehow&amp;gl=US">Related Searches</a>
</h2>
<p>Promoted By Zergnet</p>
</article>
</div>
</span>
</span>
<h2>
<a target="_blank"
href="https://www.google.com/adsense/support/bin/request.py?contact=abg_afc&amp;url=http://ehow.com/&amp;hl=en&amp;client=ehow&amp;gl=US">Related
Searches</a>
</h2>
<p>Promoted By Zergnet</p>
</article>
</div>
</div>

1836
app/src/main/assets/readability/test/test-pages/ehow-2/source.html
View File

File diff suppressed because it is too large Load Diff

34
app/src/main/assets/readability/test/test-pages/embedded-videos/expected.html
View File

@ -1,16 +1,36 @@
<div id="readability-page-1" class="page">
<article>
<h2>Lorem</h2>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud
exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure
dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt
mollit anim id est laborum.</p>
<p>At root</p>
<iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen=""></iframe>
<iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen=""></iframe>
<iframe src="https://player.vimeo.com/video/32246206?color=ffffff+title=0+byline=0+portrait=0" width="500" height="281" frameborder="0" webkitallowfullscreen="" mozallowfullscreen="" allowfullscreen=""></iframe>
<iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU"
frameborder="0" allowfullscreen=""></iframe>
<iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU"
frameborder="0" allowfullscreen=""></iframe>
<iframe src="https://player.vimeo.com/video/32246206?color=ffffff+title=0+byline=0+portrait=0"
width="500" height="281" frameborder="0" webkitallowfullscreen=""
mozallowfullscreen="" allowfullscreen=""></iframe>
<p>In a paragraph</p>
<p><iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen=""></iframe></p>
<p>
<iframe width="560" height="315"
src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU" frameborder="0"
allowfullscreen=""></iframe>
</p>
<p>In a div</p>
<p><iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU" frameborder="0" allowfullscreen=""></iframe></p>
<p>
<iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU"
frameborder="0" allowfullscreen=""></iframe>
</p>
<h2>Foo</h2>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. </p>
<p> Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute
irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla
pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia
deserunt mollit anim id est laborum. </p>
</article>
</div>

42
app/src/main/assets/readability/test/test-pages/embedded-videos/source.html
View File

@ -1,19 +1,19 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Embedded videos test</title>
<meta charset="utf-8"/>
<title>Embedded videos test</title>
</head>
<body>
<article>
<article>
<h1>Lorem</h1>
<div>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
</div>
<h2>Videos</h2>
<p>At root</p>
@ -25,19 +25,23 @@
width="500" height="281" frameborder="0"
webkitallowfullscreen="" mozallowfullscreen="" allowfullscreen=""></iframe>
<p>In a paragraph</p>
<p><iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU"
frameborder="0" allowfullscreen=""></iframe></p>
<p>
<iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/LtOGa5M8AuU"
frameborder="0" allowfullscreen=""></iframe>
</p>
<p>In a div</p>
<div><iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU"
frameborder="0" allowfullscreen=""></iframe></div>
<div>
<iframe width="560" height="315" src="https://www.youtube.com/embed/LtOGa5M8AuU"
frameborder="0" allowfullscreen=""></iframe>
</div>
<h2>Foo</h2>
<div>
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
</article>
</article>
</body>
</html>

228
app/src/main/assets/readability/test/test-pages/engadget/expected.html
View File

@ -1,114 +1,252 @@
<div id="readability-page-1" class="page">
<div>
<p>The <a href="https://www.engadget.com/2017/06/13/the-xbox-one-x-is-aspirational-in-the-purest-sense-of-the-word/">Xbox One X</a> is the ultimate video game system. It sports more horsepower than any system ever. And it plays more titles in native 4K than <a href="https://www.engadget.com/2016/11/07/sony-playstation-4-pro-review/">Sony's PlayStation 4 Pro</a>. It's just about everything you could want without investing in a gaming PC. The only problem? It's now been a year since the PS4 Pro launched, and the One X costs $500, while Sony's console launched at $400. That high price limits the Xbox One X to diehard Microsoft fans who don't mind paying a bit more to play the console's exclusive titles in 4K. Everyone else might be better off waiting, or opting for the $279 <a href="https://www.engadget.com/2016/08/02/xbox-one-s-review/">Xbox One S</a>. </p>
<p>The <a
href="https://www.engadget.com/2017/06/13/the-xbox-one-x-is-aspirational-in-the-purest-sense-of-the-word/">Xbox
One X</a> is the ultimate video game system. It sports more horsepower than any system
ever. And it plays more titles in native 4K than <a
href="https://www.engadget.com/2016/11/07/sony-playstation-4-pro-review/">Sony's
PlayStation 4 Pro</a>. It's just about everything you could want without investing
in a gaming PC. The only problem? It's now been a year since the PS4 Pro launched, and
the One X costs $500, while Sony's console launched at $400. That high price limits the
Xbox One X to diehard Microsoft fans who don't mind paying a bit more to play the
console's exclusive titles in 4K. Everyone else might be better off waiting, or opting
for the $279 <a href="https://www.engadget.com/2016/08/02/xbox-one-s-review/">Xbox One
S</a>. </p>
</div>
<section>
<h4> Gallery: Xbox One X | 14 Photos </h4>
<div data-behavior="lightbox_trigger" data-engadget-slideshow-id="803271" data-eng-bang="{&quot;gallery&quot;:803271,&quot;slide&quot;:7142088,&quot;index&quot;:0}" data-eng-mn="93511844">
<p><a href="#" data-index="0" data-engadget-slide-id="7142088" data-eng-bang="{&quot;gallery&quot;:803271,&quot;slide&quot;:7142088,&quot;index&quot;:0}">
<img src="https://o.aolcdn.com/images/dims?thumbnail=980%2C653&amp;quality=80&amp;image_uri=https%3A%2F%2Fs.blogcdn.com%2Fslideshows%2Fimages%2Fslides%2F714%2F208%2F8%2FS7142088%2Fslug%2Fl%2Fxbox-one-x-review-gallery-1-1.jpg&amp;client=cbc79c14efcebee57402&amp;signature=9bb08b52e12de8e4060f863a52c613489529818d" />
</a></p>
<div data-behavior="lightbox_trigger" data-engadget-slideshow-id="803271"
data-eng-bang="{&quot;gallery&quot;:803271,&quot;slide&quot;:7142088,&quot;index&quot;:0}"
data-eng-mn="93511844">
<p><a href="#" data-index="0" data-engadget-slide-id="7142088"
data-eng-bang="{&quot;gallery&quot;:803271,&quot;slide&quot;:7142088,&quot;index&quot;:0}">
<img src="https://o.aolcdn.com/images/dims?thumbnail=980%2C653&amp;quality=80&amp;image_uri=https%3A%2F%2Fs.blogcdn.com%2Fslideshows%2Fimages%2Fslides%2F714%2F208%2F8%2FS7142088%2Fslug%2Fl%2Fxbox-one-x-review-gallery-1-1.jpg&amp;client=cbc79c14efcebee57402&amp;signature=9bb08b52e12de8e4060f863a52c613489529818d"/>
</a></p>
</div>
</section>
<div>
<div>
<div>
<ul>
<li>Most powerful hardware ever in a home console </li>
<li>Solid selection of enhanced titles </li>
<li>4K Blu-ray drive is great for movie fans </li>
<li>Most powerful hardware ever in a home console</li>
<li>Solid selection of enhanced titles</li>
<li>4K Blu-ray drive is great for movie fans</li>
</ul>
</div>
<div>
<ul>
<li>Expensive </li>
<li>Not worth it if you don’t have a 4K TV </li>
<li>Still no VR support </li>
<li>Expensive</li>
<li>Not worth it if you don’t have a 4K TV</li>
<li>Still no VR support</li>
</ul>
</div>
</div>
<div>
<p>As promised, the Xbox One X is the most powerful game console ever. In practice, though, it really just puts Microsoft on equal footing with Sony’s PlayStation 4 Pro. 4K/HDR enhanced games look great, but it’s lack of VR is disappointing in 2017.</p>
<p>As promised, the Xbox One X is the most powerful game console ever. In practice,
though, it really just puts Microsoft on equal footing with Sony’s PlayStation 4
Pro. 4K/HDR enhanced games look great, but it’s lack of VR is disappointing in
2017.</p>
</div>
</div>
<div>
<div>
<h3>Hardware</h3>
<p><img data-credit="Devindra Hardawar/AOL" data-mep="2181678" src="https://o.aolcdn.com/images/dims?crop=1600%2C1067%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C1067&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F93beb86758ae1cf95721699e1e006e35%2F205826074%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B7.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=c0f2d36259c2c1decfb60aae364527cda2560d4a" alt="" /></p>
<p>Despite all the power inside, the One X is Microsoft's smallest console to date. It looks similar to the Xbox One S, except it has an entirely matte black case and is slightly slimmer. It's also surprisingly dense -- the console weighs 8.4 pounds, but it feels far heavier than you'd expect for its size, thanks to all of its new hardware. The One S, in comparison, weighs two pounds less.</p>
<p>The Xbox One X's real upgrades are under the hood. It features an 8-core CPU running at 2.3Ghz, 12GB of GDDR5 RAM, a 1 terabyte hard drive and an upgraded AMD Polaris GPU with 6 teraflops of computing power. The PS4 Pro has only 8GB of RAM and tops out at 4.2 teraflops. Microsoft's console is clearly faster. That additional horsepower means the Xbox One X can run more games in full native 4K than the Sony's console.</p>
<p><img data-credit="Devindra Hardawar/AOL" data-mep="2182489" src="https://o.aolcdn.com/images/dims?crop=1600%2C949%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C949&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F9ece7fdad1e7025dec06ac9bf98688d0%2F205826075%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B5.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=9913883753141e7df322616bfe0bc41c6ecd80c8" alt="" /></p>
<p>Along the front, there's the slot-loading 4K Blu-ray drive, a physical power button, a single USB port and a controller pairing button. And around back, there are HDMI out and in ports, the latter of which lets you plug in your cable box. Additionally, there are two USB ports, connections for optical audio, IR out, and gigabit Ethernet. If you've still got a Kinect around, you'll need to use a USB adapter to plug it in.</p>
<p><img data-credit="Devindra Hardawar/AOL" data-mep="2181678"
src="https://o.aolcdn.com/images/dims?crop=1600%2C1067%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C1067&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F93beb86758ae1cf95721699e1e006e35%2F205826074%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B7.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=c0f2d36259c2c1decfb60aae364527cda2560d4a"
alt=""/></p>
<p>Despite all the power inside, the One X is Microsoft's smallest console to date. It
looks similar to the Xbox One S, except it has an entirely matte black case and is
slightly slimmer. It's also surprisingly dense -- the console weighs 8.4 pounds, but
it feels far heavier than you'd expect for its size, thanks to all of its new
hardware. The One S, in comparison, weighs two pounds less.</p>
<p>The Xbox One X's real upgrades are under the hood. It features an 8-core CPU running
at 2.3Ghz, 12GB of GDDR5 RAM, a 1 terabyte hard drive and an upgraded AMD Polaris
GPU with 6 teraflops of computing power. The PS4 Pro has only 8GB of RAM and tops
out at 4.2 teraflops. Microsoft's console is clearly faster. That additional
horsepower means the Xbox One X can run more games in full native 4K than the Sony's
console.</p>
<p><img data-credit="Devindra Hardawar/AOL" data-mep="2182489"
src="https://o.aolcdn.com/images/dims?crop=1600%2C949%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C949&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F9ece7fdad1e7025dec06ac9bf98688d0%2F205826075%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B5.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=9913883753141e7df322616bfe0bc41c6ecd80c8"
alt=""/></p>
<p>Along the front, there's the slot-loading 4K Blu-ray drive, a physical power button,
a single USB port and a controller pairing button. And around back, there are HDMI
out and in ports, the latter of which lets you plug in your cable box. Additionally,
there are two USB ports, connections for optical audio, IR out, and gigabit
Ethernet. If you've still got a Kinect around, you'll need to use a USB adapter to
plug it in.</p>
</div>
<div data-engadget-breakout-type="image">
<figure><img src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1599%252C1043%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C1043%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252F8b98ec8f6649158fe7448ac2f2695ac5%252F205826072%252FXbox%252BOne%252BX%252Breview%252Bgallery%252B6.jpg%26client%3Da1acac3e1b3290917d92%26signature%3D353dad1308f98c2c9dfc82c58a540a8b2f1fe63c&amp;client=cbc79c14efcebee57402&amp;signature=60b7c061460d0d45f5d367b8a9c62978af6b76ce" />
<figure><img
src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1599%252C1043%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C1043%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252F8b98ec8f6649158fe7448ac2f2695ac5%252F205826072%252FXbox%252BOne%252BX%252Breview%252Bgallery%252B6.jpg%26client%3Da1acac3e1b3290917d92%26signature%3D353dad1308f98c2c9dfc82c58a540a8b2f1fe63c&amp;client=cbc79c14efcebee57402&amp;signature=60b7c061460d0d45f5d367b8a9c62978af6b76ce"/>
<figcaption><span>Devindra Hardawar/AOL</span>
</figcaption>
</figure>
</div>
<div>
<p>The console's controller hasn't changed since its last mini-upgrade with the Xbox One S. That revision rounded out its seams, improved bumper performance and added a 3.5mm headphone jack. It's still a great controller, though I'm annoyed Microsoft is sticking with AA batteries as their default power source. Sure, you could just pick up some renewable batteries, or the Play and Charge kit, but that's an extra expense. And manually swapping batteries feels like a bad user experience when every other console has rechargeable controllers.</p>
<p>The console's controller hasn't changed since its last mini-upgrade with the Xbox One
S. That revision rounded out its seams, improved bumper performance and added a
3.5mm headphone jack. It's still a great controller, though I'm annoyed Microsoft is
sticking with AA batteries as their default power source. Sure, you could just pick
up some renewable batteries, or the Play and Charge kit, but that's an extra
expense. And manually swapping batteries feels like a bad user experience when every
other console has rechargeable controllers.</p>
<h3>In use</h3>
</div>
<div data-engadget-breakout-type="image">
<figure><img src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1600%252C900%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C900%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252F1885534bd201fc37481b806645c1fc8b%252F205828119%252FXbox%252Bone%252BX%252Bscreenshot%252Bgallery%252B8.jpg%26client%3Da1acac3e1b3290917d92%26signature%3Df63cf67c88b37fd9424855984e45f6b950c8c11a&amp;client=cbc79c14efcebee57402&amp;signature=0adca80fc8ee26a7353be639082881450a5ad49f" />
<figure><img
src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1600%252C900%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C900%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252F1885534bd201fc37481b806645c1fc8b%252F205828119%252FXbox%252Bone%252BX%252Bscreenshot%252Bgallery%252B8.jpg%26client%3Da1acac3e1b3290917d92%26signature%3Df63cf67c88b37fd9424855984e45f6b950c8c11a&amp;client=cbc79c14efcebee57402&amp;signature=0adca80fc8ee26a7353be639082881450a5ad49f"/>
<figcaption><span>Devindra Hardawar/AOL</span>
</figcaption>
</figure>
</div>
<div>
<p>You won't find any major differences between the One X and the last Xbox at first — aside from a more dramatic startup sequence. Navigating the Xbox interface is fast and zippy, but mostly that's due to a recent OS upgrade. If you're moving over from an older Xbox One, you can use the backup tool to transfer your games and settings to an external hard drive. Just plug that into the new console during setup and it'll make it feel just like your old machine. It's also a lot faster than waiting for everything to download from Xbox Live.</p>
<p>You'll still have to set aside some time if you want to play an Xbox One X-enhanced title, though. Those 4K textures will make games significantly larger, but Microsoft says it's come up with a few ways to help developers make downloading them more efficient. For example, language packs and other optional content won't get installed by default.</p>
<p>We only had a few enhanced titles to test out during our review: <em>Gears of War 4</em>, <em>Killer Instinct</em> and <em>Super Lucky's Tale</em>. They each took advantage of the console in different ways. <em>Gears of War 4</em> runs natively in 4K at 30 FPS with Dolby Atmos and HDR (high dynamic range lighting) support. It looked great -- especially with HDR, which highlighted bright elements like lightning strikes -- but I noticed the frame rate dip occasionally. I was also surprised that load times were on-par with what I've seen with the game on the Xbox One S.</p>
<p>You won't find any major differences between the One X and the last Xbox at first —
aside from a more dramatic startup sequence. Navigating the Xbox interface is fast
and zippy, but mostly that's due to a recent OS upgrade. If you're moving over from
an older Xbox One, you can use the backup tool to transfer your games and settings
to an external hard drive. Just plug that into the new console during setup and
it'll make it feel just like your old machine. It's also a lot faster than waiting
for everything to download from Xbox Live.</p>
<p>You'll still have to set aside some time if you want to play an Xbox One X-enhanced
title, though. Those 4K textures will make games significantly larger, but Microsoft
says it's come up with a few ways to help developers make downloading them more
efficient. For example, language packs and other optional content won't get
installed by default.</p>
<p>We only had a few enhanced titles to test out during our review: <em>Gears of War
4</em>, <em>Killer Instinct</em> and <em>Super Lucky's Tale</em>. They each took
advantage of the console in different ways. <em>Gears of War 4</em> runs natively in
4K at 30 FPS with Dolby Atmos and HDR (high dynamic range lighting) support. It
looked great -- especially with HDR, which highlighted bright elements like
lightning strikes -- but I noticed the frame rate dip occasionally. I was also
surprised that load times were on-par with what I've seen with the game on the Xbox
One S.</p>
</div>
<div data-engadget-breakout-type="e2ehero">
<figure><img src="https://o.aolcdn.com/images/dims?crop=1600%2C900%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C900&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F8352a8a14e88e2ca2ba5be4d8381a055%2F205828115%2FXbox%2Bone%2BX%2Bscreenshot%2Bgallery%2B1.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=d2ccb22e0eaabeb05bfe46e83dbe26fd07f01da8" />
<figure><img
src="https://o.aolcdn.com/images/dims?crop=1600%2C900%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C900&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F8352a8a14e88e2ca2ba5be4d8381a055%2F205828115%2FXbox%2Bone%2BX%2Bscreenshot%2Bgallery%2B1.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=d2ccb22e0eaabeb05bfe46e83dbe26fd07f01da8"/>
</figure>
</div>
<div>
<p>You can also play in Performance mode, which bumps the frame rate up to 60FPS and uses higher quality graphical effects, while rendering it lower in 1080p. Personally, I preferred this, since it makes the game much smoother -- as if you're playing it on a high-end gaming PC, not a console. Some PlayStation 4 Pro games also let you choose how you wanted to distribute its power, so in some ways Microsoft is just following in its footsteps.</p>
<p>I've been playing <em>Gears of War 4</em> on my gaming PC (which is connected to my home theater) over the past year, and I was impressed that the Xbox One X is able to deliver a similar experience. It didn't quite match my rig though, which is powered by Intel Core i7 4790k CPU running at 4GHz, 16GB DDR3 RAM and an NVIDIA GTX 1080 GPU. Typically, I play at 1,440p (2,560 by 1,440 pixels) with HDR and all of the graphical settings set to their highest level, and I can easily maintain a 60FPS frame rate. The One X felt just as solid at 1080p, but there were clearly plenty of graphics settings it couldn't take advantage of, in particular higher levels of bloom lighting and shadow detail.</p>
<p>You can also play in Performance mode, which bumps the frame rate up to 60FPS and
uses higher quality graphical effects, while rendering it lower in 1080p.
Personally, I preferred this, since it makes the game much smoother -- as if you're
playing it on a high-end gaming PC, not a console. Some PlayStation 4 Pro games also
let you choose how you wanted to distribute its power, so in some ways Microsoft is
just following in its footsteps.</p>
<p>I've been playing <em>Gears of War 4</em> on my gaming PC (which is connected to my
home theater) over the past year, and I was impressed that the Xbox One X is able to
deliver a similar experience. It didn't quite match my rig though, which is powered
by Intel Core i7 4790k CPU running at 4GHz, 16GB DDR3 RAM and an NVIDIA GTX 1080
GPU. Typically, I play at 1,440p (2,560 by 1,440 pixels) with HDR and all of the
graphical settings set to their highest level, and I can easily maintain a 60FPS
frame rate. The One X felt just as solid at 1080p, but there were clearly plenty of
graphics settings it couldn't take advantage of, in particular higher levels of
bloom lighting and shadow detail.</p>
</div>
<section data-engadget-breakout-type="gallery">
<h3> Gallery: Xbox One X screenshots | 9 Photos </h3>
<div data-behavior="lightbox_trigger" data-engadget-slideshow-id="803330" data-eng-bang="{&quot;gallery&quot;:803330,&quot;slide&quot;:7142924}" data-eng-mn="93511844">
<p><a href="#" data-index="0" data-engadget-slide-id="7142924" data-eng-bang="{&quot;gallery&quot;:803330,&quot;slide&quot;:7142924}">
<img src="https://o.aolcdn.com/images/dims?thumbnail=980%2C653&amp;quality=80&amp;image_uri=https%3A%2F%2Fs.blogcdn.com%2Fslideshows%2Fimages%2Fslides%2F714%2F292%2F4%2FS7142924%2Fslug%2Fl%2Fxbox-one-x-screenshot-gallery-2-1.jpg&amp;client=cbc79c14efcebee57402&amp;signature=38c95635c7aad58a8a48038e05589f5cf35b1e28" />
</a></p>
<div data-behavior="lightbox_trigger" data-engadget-slideshow-id="803330"
data-eng-bang="{&quot;gallery&quot;:803330,&quot;slide&quot;:7142924}"
data-eng-mn="93511844">
<p><a href="#" data-index="0" data-engadget-slide-id="7142924"
data-eng-bang="{&quot;gallery&quot;:803330,&quot;slide&quot;:7142924}">
<img src="https://o.aolcdn.com/images/dims?thumbnail=980%2C653&amp;quality=80&amp;image_uri=https%3A%2F%2Fs.blogcdn.com%2Fslideshows%2Fimages%2Fslides%2F714%2F292%2F4%2FS7142924%2Fslug%2Fl%2Fxbox-one-x-screenshot-gallery-2-1.jpg&amp;client=cbc79c14efcebee57402&amp;signature=38c95635c7aad58a8a48038e05589f5cf35b1e28"/>
</a></p>
</div>
</section>
<div>
<p><em>Killer Instinct</em> and <em>Super Lucky's Tale</em> run in 4K at a smooth 60FPS. They both looked and played better than their standard versions, though I was surprised they didn't take advantage of HDR. As usual, I noticed the improvement in frame rates more than the higher resolution. Unless you're sitting very close to a TV above 50-inches, you'd likely have a hard time telling between 4K and 1080p.</p>
<p>That poses a problem for Microsoft: It's betting that gamers will actually want true 4K rendering. In practice, though, PlayStation 4 Pro titles running in HDR and resolutions between 1080p and 4K often look just as good to the naked eye. The Xbox One X's big advantage is that its hardware could let more games reach 60FPS compared to Sony's console.</p>
<p>Microsoft says over 130 Xbox One X-enhanced titles are in the works. That includes already-released games like <em>Forza Motorsport 7</em> and <em>Assassin's Creed Origins</em>, as well as upcoming titles like <em>Call of Duty: WW2</em>. You'll be able to find them easily in a special section in the Xbox store. There is also a handful of Xbox 360 games that'll get enhanced eventually, including <em>Halo 3</em> and <em>Fallout 3</em>. Some of those titles will get bumped up to a higher resolution, while others will get HDR support. Microsoft describes these upgrades as a bonus for developers who were prescient about how they built their games. Basically, don't expect your entire 360 library to get enhanced.</p>
<p><em>Killer Instinct</em> and <em>Super Lucky's Tale</em> run in 4K at a smooth 60FPS.
They both looked and played better than their standard versions, though I was
surprised they didn't take advantage of HDR. As usual, I noticed the improvement in
frame rates more than the higher resolution. Unless you're sitting very close to a
TV above 50-inches, you'd likely have a hard time telling between 4K and 1080p.</p>
<p>That poses a problem for Microsoft: It's betting that gamers will actually want true
4K rendering. In practice, though, PlayStation 4 Pro titles running in HDR and
resolutions between 1080p and 4K often look just as good to the naked eye. The Xbox
One X's big advantage is that its hardware could let more games reach 60FPS compared
to Sony's console.</p>
<p>Microsoft says over 130 Xbox One X-enhanced titles are in the works. That includes
already-released games like <em>Forza Motorsport 7</em> and <em>Assassin's Creed
Origins</em>, as well as upcoming titles like <em>Call of Duty: WW2</em>. You'll
be able to find them easily in a special section in the Xbox store. There is also a
handful of Xbox 360 games that'll get enhanced eventually, including <em>Halo 3</em>
and <em>Fallout 3</em>. Some of those titles will get bumped up to a higher
resolution, while others will get HDR support. Microsoft describes these upgrades as
a bonus for developers who were prescient about how they built their games.
Basically, don't expect your entire 360 library to get enhanced.</p>
</div>
<div data-engadget-breakout-type="e2ehero">
<figure><img src="https://o.aolcdn.com/images/dims?crop=1600%2C900%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C900&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2Facb08903fbe26ad77b80db8c8e7e8fb1%2F205828118%2FXbox%2Bone%2BX%2Bscreenshot%2Bgallery%2B7.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=21630fa5ec6d8fdce2c35f7e1f652636a2d8efe7" />
<figure><img
src="https://o.aolcdn.com/images/dims?crop=1600%2C900%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C900&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2Facb08903fbe26ad77b80db8c8e7e8fb1%2F205828118%2FXbox%2Bone%2BX%2Bscreenshot%2Bgallery%2B7.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=21630fa5ec6d8fdce2c35f7e1f652636a2d8efe7"/>
</figure>
</div>
<div>
<p>Even if a game isn't specifically tuned for the new console, Microsoft says you might still see some performance improvements. The PlayStation 4 Pro, meanwhile, has over one hundred games built for its hardware, and its boost mode can speed up some older games.</p>
<p>Microsoft is still pushing the Xbox as more than just a game console, though. 4K Blu-rays loaded up quickly, and I didn't notice many delays as I skipped around films. <em>Planet Earth II</em>, in particular, looked fantastic thanks to its brilliant use of HDR. Unfortunately, the One X doesn't support Dolby Vision, so you're stuck with the slightly less capable HDR 10 standard. That makes sense since it's more widely supported, but it would have been nice to see Dolby's, too.</p>
<p>Even if a game isn't specifically tuned for the new console, Microsoft says you might
still see some performance improvements. The PlayStation 4 Pro, meanwhile, has over
one hundred games built for its hardware, and its boost mode can speed up some older
games.</p>
<p>Microsoft is still pushing the Xbox as more than just a game console, though. 4K
Blu-rays loaded up quickly, and I didn't notice many delays as I skipped around
films. <em>Planet Earth II</em>, in particular, looked fantastic thanks to its
brilliant use of HDR. Unfortunately, the One X doesn't support Dolby Vision, so
you're stuck with the slightly less capable HDR 10 standard. That makes sense since
it's more widely supported, but it would have been nice to see Dolby's, too.</p>
<p>
<iframe allowfullscreen="" frameborder="0" gesture="media" height="360" src="https://www.youtube.com/embed/c8aFcHFu8QM" width="640"></iframe>
<iframe allowfullscreen="" frameborder="0" gesture="media" height="360"
src="https://www.youtube.com/embed/c8aFcHFu8QM" width="640"></iframe>
</p>
<p>And speaking of Dolby technology, Microsoft is also highlighting Atmos support on the One X, just like it did with the One S. The company's app lets you configure the console to pass audio Atmos signals to your audio receiver. You can also shell out $15 to get Atmos support for headphones, which simulates immersive surround sound. It's strange to pay money to unlock Dolby features, but it's worth it since it's significantly better than Microsoft's audio virtualization technology. The Netflix app also supports Atmos for a handful of films (something that the Xbox One S and PlayStation 4 offer, as well).</p>
<p>One thing you won't find in the new Xbox is VR support. Microsoft has mentioned that the console will offer some sort of mixed reality, but it hasn't offered up any details yet. It's technically powerful enough to work with any of the Windows Mixed Reality headsets launching this fall. It's a shame that Microsoft is being so wishy-washy because Sony has had a very successful head start with the PlayStation VR.</p>
<p>And speaking of Dolby technology, Microsoft is also highlighting Atmos support on the
One X, just like it did with the One S. The company's app lets you configure the
console to pass audio Atmos signals to your audio receiver. You can also shell out
$15 to get Atmos support for headphones, which simulates immersive surround sound.
It's strange to pay money to unlock Dolby features, but it's worth it since it's
significantly better than Microsoft's audio virtualization technology. The Netflix
app also supports Atmos for a handful of films (something that the Xbox One S and
PlayStation 4 offer, as well).</p>
<p>One thing you won't find in the new Xbox is VR support. Microsoft has mentioned that
the console will offer some sort of mixed reality, but it hasn't offered up any
details yet. It's technically powerful enough to work with any of the Windows Mixed
Reality headsets launching this fall. It's a shame that Microsoft is being so
wishy-washy because Sony has had a very successful head start with the PlayStation
VR.</p>
<h3>Pricing and the competition</h3>
</div>
<div data-engadget-breakout-type="image">
<figure><img src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1600%252C1027%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C1027%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252Fa2c8ba1caccdbb9e0559797e5141eafd%252F205826078%252FXbox%252BOne%252BX%252Breview%252Bgallery%252B11.jpg%26client%3Da1acac3e1b3290917d92%26signature%3Da11bcddced805c6e3698f8ce0494102aef057265&amp;client=cbc79c14efcebee57402&amp;signature=1e9bd192add2772bc842a34e67b7572cfd1b265a" />
<figure><img
src="https://o.aolcdn.com/images/dims?resize=980%2C640&amp;quality=100&amp;image_uri=https%3A%2F%2Fo.aolcdn.com%2Fimages%2Fdims%3Fcrop%3D1600%252C1027%252C0%252C0%26quality%3D85%26format%3Djpg%26resize%3D1600%252C1027%26image_uri%3Dhttp%253A%252F%252Fo.aolcdn.com%252Fhss%252Fstorage%252Fmidas%252Fa2c8ba1caccdbb9e0559797e5141eafd%252F205826078%252FXbox%252BOne%252BX%252Breview%252Bgallery%252B11.jpg%26client%3Da1acac3e1b3290917d92%26signature%3Da11bcddced805c6e3698f8ce0494102aef057265&amp;client=cbc79c14efcebee57402&amp;signature=1e9bd192add2772bc842a34e67b7572cfd1b265a"/>
<figcaption><span>Devindra Hardawar/AOL</span>
</figcaption>
</figure>
</div>
<div>
<p>The biggest knock against the Xbox One X is its $500 price. The PS4 Pro launched at $400 last year, and there's a good chance we'll see plenty of deals around the holidays. If your friends are on Xbox Live, or you're a devotee of Microsoft's first party franchises, then the X makes more sense. If you just want to play third-party titles that come to both platforms, though, the PS4 Pro is clearly the better deal.</p>
<p>If you're looking to upgrade from an original Xbox One, and you have a new TV, the One X might be more compelling. It's faster and offers more features than the One S, and more importantly, it'll last you much longer without needing an upgrade. There's also plenty of wisdom in simply waiting a while before you buy the One X, especially if you haven't moved to a 4K TV yet. The new console can make games look better on 1080p sets, since it'll supersample high-res textures and have more graphical effects, but it's simply not worth the upgrade since those TVs don't support HDR.</p>
<p>If price isn't a huge concern for you, it's worth considering investing in a gaming PC. A decent one costs between $600 and $800, plus the price of a monitor, but it'll easily be more powerful than the One X. And you have the added benefit of upgrading components down the line. Now that Microsoft and game publishers are offering most major titles on PC, you won't be missing out on much by ditching consoles.</p>
<p>The biggest knock against the Xbox One X is its $500 price. The PS4 Pro launched at
$400 last year, and there's a good chance we'll see plenty of deals around the
holidays. If your friends are on Xbox Live, or you're a devotee of Microsoft's first
party franchises, then the X makes more sense. If you just want to play third-party
titles that come to both platforms, though, the PS4 Pro is clearly the better
deal.</p>
<p>If you're looking to upgrade from an original Xbox One, and you have a new TV, the
One X might be more compelling. It's faster and offers more features than the One S,
and more importantly, it'll last you much longer without needing an upgrade. There's
also plenty of wisdom in simply waiting a while before you buy the One X, especially
if you haven't moved to a 4K TV yet. The new console can make games look better on
1080p sets, since it'll supersample high-res textures and have more graphical
effects, but it's simply not worth the upgrade since those TVs don't support
HDR.</p>
<p>If price isn't a huge concern for you, it's worth considering investing in a gaming
PC. A decent one costs between $600 and $800, plus the price of a monitor, but it'll
easily be more powerful than the One X. And you have the added benefit of upgrading
components down the line. Now that Microsoft and game publishers are offering most
major titles on PC, you won't be missing out on much by ditching consoles.</p>
<h3>Wrap-up</h3>
<p><img data-credit="Devindra Hardawar/AOL" data-mep="2181681" src="https://o.aolcdn.com/images/dims?crop=1600%2C1028%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C1028&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F5396460ef8b6bde7fb7272d9e66a7701%2F205826076%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B9.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=f5b5b4b986c2f8b5031a4469ae0ecec82aff65b0" alt="" /></p>
<p>Ultimately, the Xbox One X offers some major performance upgrades that gamers will notice -- especially if you're coming from an original Xbox One. But it's also a bit disappointing since it's coming a year after the PS4 Pro, and it doesn't offer VR yet. For Microsoft fans, though, none of that will matter. It's exactly what the company promised: the fastest game console ever made.</p>
<p><img data-credit="Devindra Hardawar/AOL" data-mep="2181681"
src="https://o.aolcdn.com/images/dims?crop=1600%2C1028%2C0%2C0&amp;quality=85&amp;format=jpg&amp;resize=1600%2C1028&amp;image_uri=http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2F5396460ef8b6bde7fb7272d9e66a7701%2F205826076%2FXbox%2BOne%2BX%2Breview%2Bgallery%2B9.jpg&amp;client=a1acac3e1b3290917d92&amp;signature=f5b5b4b986c2f8b5031a4469ae0ecec82aff65b0"
alt=""/></p>
<p>Ultimately, the Xbox One X offers some major performance upgrades that gamers will
notice -- especially if you're coming from an original Xbox One. But it's also a bit
disappointing since it's coming a year after the PS4 Pro, and it doesn't offer VR
yet. For Microsoft fans, though, none of that will matter. It's exactly what the
company promised: the fastest game console ever made.</p>
</div>
</div>
</div>

2496
app/src/main/assets/readability/test/test-pages/engadget/source.html
View File

File diff suppressed because one or more lines are too long

43
app/src/main/assets/readability/test/test-pages/folha/expected.html
View File

@ -1,11 +1,38 @@
<div id="readability-page-1" class="page">
<div data-share-text="" data-news-content-text="" data-disable-copy="" data-continue-reading="" data-continue-reading-hide-others=".js-continue-reading-hidden" itemprop="articleBody">
<p> Após rechaçar <a href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml">um encontro da seleção brasileira com o presidente eleito Jair</a> <a href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml">Bolsonaro</a>, o técnico Tite declarou que errou ao levar a taça da Copa Libertadores de 2012, conquistada pelo Corinthians, ao ex-presidente Luiz Inácio Lula da Silva. </p>
<p> Ao lado de representantes do clube paulista, o atual comandante do Brasil ainda entregou uma réplica do troféu a Lula. </p>
<p> "Em 2012 eu errei. Ele não era presidente, mas fui ao Instituto e mandei felicitações por um aniversário. Não me posicionei politicamente. Não tenho partido político, tenho sim a torcida para que o Brasil seja melhor em igualdade social. E que nossas prioridades sejam educação e punição. Que seja dada a possibilidade de estudo ao garoto de São Braz, que não tem chão batido para ir à escola, ou da periferia de Caixas ou do morro do Rio de Janeiro. Seja dada a ele a prioridade de estudo e não a outras situações", falou Tite ao programa "Grande Círculo", que ainda irá ao ar no SporTV. </p>
<p> Na ocasião, Tite e outros representantes do Corinthians <a href="https://www1.folha.uol.com.br/poder/1124743-corinthians-leva-a-taca-da-libertadores-para-lula.shtml">foram ao Instituto Lula para mostrar a taça</a> original da Libertadores ao ex-presidente. </p>
<p> O assunto foi levantado&#160;porque recentemente Tite foi questionado se aceitaria um encontro da seleção brasileira com Bolsonaro em uma conquista de título ou <a href="https://www1.folha.uol.com.br/esporte/2018/12/selecao-brasileira-jogara-duas-vezes-em-sao-paulo-na-copa-america.shtml">antes da Copa América de 2019</a>, por exemplo. O treinador deixou claro que preferiria evitar esse tipo de formalidade. </p>
<p> Apesar disso, Tite não questionou a ação de Palmeiras e CBF, que <a href="https://www1.folha.uol.com.br/esporte/2018/12/cbf-usa-festa-do-palmeiras-para-se-aproximar-de-governo-bolsonaro.shtml">convidaram Bolsonaro para a festa do título do Campeonato Brasileiro</a>. O presidente eleito até levantou a taça conquistada pelo clube alviverde. </p>
<p> "Em 2012 eu fiz e errei. O protocolo e a situação gerada no jogo do Palmeiras são fatos de opinião pessoal. CBF e Palmeiras, enquanto instituições têm a opinião. Errei lá atrás, não faria com o presidente antes da Copa e nem agora porque entendo que misturar esporte e política não é legal. Fiz errado lá atrás? Sim. Faria de novo? Não", acrescentou o comandante. </p>
<div data-share-text="" data-news-content-text="" data-disable-copy="" data-continue-reading=""
data-continue-reading-hide-others=".js-continue-reading-hidden" itemprop="articleBody">
<p> Após rechaçar <a
href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml">um
encontro da seleção brasileira com o presidente eleito Jair</a> <a
href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml">Bolsonaro</a>,
o técnico Tite declarou que errou ao levar a taça da Copa Libertadores de 2012,
conquistada pelo Corinthians, ao ex-presidente Luiz Inácio Lula da Silva. </p>
<p> Ao lado de representantes do clube paulista, o atual comandante do Brasil ainda entregou
uma réplica do troféu a Lula. </p>
<p> "Em 2012 eu errei. Ele não era presidente, mas fui ao Instituto e mandei felicitações
por um aniversário. Não me posicionei politicamente. Não tenho partido político, tenho
sim a torcida para que o Brasil seja melhor em igualdade social. E que nossas
prioridades sejam educação e punição. Que seja dada a possibilidade de estudo ao garoto
de São Braz, que não tem chão batido para ir à escola, ou da periferia de Caixas ou do
morro do Rio de Janeiro. Seja dada a ele a prioridade de estudo e não a outras
situações", falou Tite ao programa "Grande Círculo", que ainda irá ao ar no SporTV. </p>
<p> Na ocasião, Tite e outros representantes do Corinthians <a
href="https://www1.folha.uol.com.br/poder/1124743-corinthians-leva-a-taca-da-libertadores-para-lula.shtml">foram
ao Instituto Lula para mostrar a taça</a> original da Libertadores ao ex-presidente.
</p>
<p> O assunto foi levantado&#160;porque recentemente Tite foi questionado se aceitaria um
encontro da seleção brasileira com Bolsonaro em uma conquista de título ou <a
href="https://www1.folha.uol.com.br/esporte/2018/12/selecao-brasileira-jogara-duas-vezes-em-sao-paulo-na-copa-america.shtml">antes
da Copa América de 2019</a>, por exemplo. O treinador deixou claro que preferiria
evitar esse tipo de formalidade. </p>
<p> Apesar disso, Tite não questionou a ação de Palmeiras e CBF, que <a
href="https://www1.folha.uol.com.br/esporte/2018/12/cbf-usa-festa-do-palmeiras-para-se-aproximar-de-governo-bolsonaro.shtml">convidaram
Bolsonaro para a festa do título do Campeonato Brasileiro</a>. O presidente eleito até
levantou a taça conquistada pelo clube alviverde. </p>
<p> "Em 2012 eu fiz e errei. O protocolo e a situação gerada no jogo do Palmeiras são fatos
de opinião pessoal. CBF e Palmeiras, enquanto instituições têm a opinião. Errei lá
atrás, não faria com o presidente antes da Copa e nem agora porque entendo que misturar
esporte e política não é legal. Fiz errado lá atrás? Sim. Faria de novo? Não",
acrescentou o comandante. </p>
</div>
</div>

5582
app/src/main/assets/readability/test/test-pages/folha/source.html
View File

File diff suppressed because one or more lines are too long

76
app/src/main/assets/readability/test/test-pages/gmw/expected.html
View File

@ -2,51 +2,81 @@
<div id="contentMain">
<p>　　翱翔于距地球数千公里的太空中，进入广袤漆黑的未知领域，是一项艰苦卓绝的工作。这让人感到巨大压力和极度恐慌。那么，为什么不能让宇航员来一杯“地球末日”鸡尾酒来放松一下？</p>
<p>　　不幸的是，对于希望能喝上一杯的太空探险者，那些将他们送上太空的政府机构普遍禁止他们染指包括酒在内的含酒精饮料。</p>
<p>　　但是，很快普通人都会有机会向人类“最终的边疆”出发——以平民化旅行的形式，去探索和殖民火星。确实，火星之旅将是一次令人感到痛苦的旅行，可能一去不复返并要几年时间才能完成，但是否应该允许参与者在旅程中痛饮一番？或至少携带能在火星上发酵自制酒精饮料的设备？</p>
<p><img id="45395168" alt="(Credit: Nasa)" src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e5929.jpg" title="宇航员在太空中喝酒会怎么样？后果很严重" /></p>
<p>
　　但是，很快普通人都会有机会向人类“最终的边疆”出发——以平民化旅行的形式，去探索和殖民火星。确实，火星之旅将是一次令人感到痛苦的旅行，可能一去不复返并要几年时间才能完成，但是否应该允许参与者在旅程中痛饮一番？或至少携带能在火星上发酵自制酒精饮料的设备？</p>
<p><img id="45395168" alt="(Credit: Nasa)"
src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e5929.jpg"
title="宇航员在太空中喝酒会怎么样？后果很严重"/></p>
<p>
<span face="楷体">　　图注：巴兹?奥尔德林(Buzz Aldrin)可能是第二个在月球上行走的人，但他是第一个在月球上喝酒的人</span>
</p>
<p>　　事实是，历史上酒与太空探险有一种复杂的关系。让我们来看看喝了酒的航天员究竟会发生什么—— 如果我们开始给予进入太空的人类更大的自由度，又可能会发生什么。</p>
<p>　　人们普遍认为，当一个人所处的海拔越高，喝醉后会越容易感到头昏。因此，人们自然地想到，当人身处地球轨道上时，饮酒会对人体有更强烈的致眩作用。但这种说法可能不是正确的。</p>
<p>　　事实上，有证据表明，早在上世纪八十年代就澄清了这一传言。1985年，美国联邦航空管理局（UFAA）开展了一项研究，以验证人在不同的海拔高度饮酒，是否会影响执行复杂任务时的表现和酒精测定仪的读数。</p>
<p>　　在这项研究中，17名男子被要求在地面和一间模拟海拔3.7公里的房间内喝下一些伏特加。然后，他们被要求完成各种任务，包括心算口算问题、用操纵杆在示波器上跟踪灯光以及各种其它测试。研究人员发现，“酒精和海拔高度对酒精测定仪读数或完成任务的表现情况没有交互作用”。</p>
<p>　　所以，人乘坐飞机时醉得更快是个传说？纽约州立大学（State University of New York，SUNY）社会学荣誉教授戴夫·汉森（Dave Hanson）研究酒精和饮酒超过40年，他认为确实如此。他说：“我不认为它（在太空中饮酒）会有任何不同。”</p>
<p>
　　事实上，有证据表明，早在上世纪八十年代就澄清了这一传言。1985年，美国联邦航空管理局（UFAA）开展了一项研究，以验证人在不同的海拔高度饮酒，是否会影响执行复杂任务时的表现和酒精测定仪的读数。</p>
<p>
　　在这项研究中，17名男子被要求在地面和一间模拟海拔3.7公里的房间内喝下一些伏特加。然后，他们被要求完成各种任务，包括心算口算问题、用操纵杆在示波器上跟踪灯光以及各种其它测试。研究人员发现，“酒精和海拔高度对酒精测定仪读数或完成任务的表现情况没有交互作用”。</p>
<p>　　所以，人乘坐飞机时醉得更快是个传说？纽约州立大学（State University of New York，SUNY）社会学荣誉教授戴夫·汉森（Dave
Hanson）研究酒精和饮酒超过40年，他认为确实如此。他说：“我不认为它（在太空中饮酒）会有任何不同。”</p>
<p>　　他认为高原反应可能类似于宿醉，但它也可能类似于中毒。他说：“如果人们没有感受到充分的大气压力，他们也会觉得喝醉了一样。”</p>
<p>　　相反，那些声称在飞机上比在地面上醉得更快的人，可能只是经历了“自认喝醉（think-drink）”效应，这种效应多年来已被广泛研究。它表明，如果人们认为自己喝醉了，那他们的一举一动会真的像喝醉了一样—— 而不是实际上他们真的醉了。</p>
<p>
　　相反，那些声称在飞机上比在地面上醉得更快的人，可能只是经历了“自认喝醉（think-drink）”效应，这种效应多年来已被广泛研究。它表明，如果人们认为自己喝醉了，那他们的一举一动会真的像喝醉了一样——
而不是实际上他们真的醉了。</p>
<p>　　汉森指出：“如果人们脑子里一直认为在飞机上酒精会对他们产生与平常不同的作用，那么他们乘坐飞机时真的会觉得酒精对他们产生了不同的作用。”</p>
<p>　　所以，如果酒精对人体的物理效应与海拔高度无关，那么在国际空间站上睡前小饮一杯不应该是一个大问题，对吧？错了。</p>
<p>　　美国宇航局约翰逊航天中心发言人丹尼尔·霍特(Daniel Huot)表示：“国际空间站上的宇航员不允许喝酒。在国际空间站上，酒精和其它挥发性化合物的使用受到控制，因为它们的挥发物可能对该站的水回收系统产生影响。”</p>
<p>　　美国宇航局约翰逊航天中心发言人丹尼尔·霍特(Daniel
Huot)表示：“国际空间站上的宇航员不允许喝酒。在国际空间站上，酒精和其它挥发性化合物的使用受到控制，因为它们的挥发物可能对该站的水回收系统产生影响。”</p>
<p>　　为此，国际空间站上的宇航员甚至没有被提供含有酒精的产品，例如漱口水、香水或须后水。如果在国际空间站上饮酒狂欢，溢出的啤酒也可能存在损坏设备的风险。</p>
<p><img id="45395150" alt="(Credit: iStock)" src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e592a.jpg" title="宇航员在太空中喝酒会怎么样？后果很严重" /></p>
<p><img id="45395150" alt="(Credit: iStock)"
src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e592a.jpg"
title="宇航员在太空中喝酒会怎么样？后果很严重"/></p>
<p>
<span face="楷体">　　图注：测试表明，有关人在高空中喝酒更容易醉的传言是不正确的</span>
</p>
<p>　　然后是责任的问题。我们不允许汽车司机或飞机飞行员喝醉后驾驶，所以并不奇怪同样的规则适用于国际空间站上的宇航员。毕竟国际空间站的造价高达1500亿美元，而且在接近真空的太空中其运行速度达到了每小时27680公里。</p>
<p>　　然而，2007年，美国宇航局(NASA)成立了一个负责调查宇航员健康状况的独立小组，称历史上该机构至少有两名宇航员在即将飞行前喝了大量的酒，但仍然被允许飞行。Nasa安全负责人随后的审查发现并没有证据支持这一指控。宇航员在飞行前12小时是严禁饮酒的，因为他们需要充分的思维能力和清醒的意识。</p>
<p>　　出台这一规则的原因很清楚。在1985年UFAA开展的关于酒精在不同海拔高度影响的研究中，研究人员得出结论，酒精的影响与海拔高度无关。无论参与测试的人员在什么海拔高度喝酒，其酒精测量仪的读数都是一样的。他们的行为表现受到的影响也相同，但如果提供给测试人员的是安慰剂，则身处高空比身处海平面的行为表现要更差一些。这表明，无论是否摄入酒精，海拔高度可能对心理表现有轻微的影响。</p>
<p>
　　然后是责任的问题。我们不允许汽车司机或飞机飞行员喝醉后驾驶，所以并不奇怪同样的规则适用于国际空间站上的宇航员。毕竟国际空间站的造价高达1500亿美元，而且在接近真空的太空中其运行速度达到了每小时27680公里。</p>
<p>
　　然而，2007年，美国宇航局(NASA)成立了一个负责调查宇航员健康状况的独立小组，称历史上该机构至少有两名宇航员在即将飞行前喝了大量的酒，但仍然被允许飞行。Nasa安全负责人随后的审查发现并没有证据支持这一指控。宇航员在飞行前12小时是严禁饮酒的，因为他们需要充分的思维能力和清醒的意识。</p>
<p>
　　出台这一规则的原因很清楚。在1985年UFAA开展的关于酒精在不同海拔高度影响的研究中，研究人员得出结论，酒精的影响与海拔高度无关。无论参与测试的人员在什么海拔高度喝酒，其酒精测量仪的读数都是一样的。他们的行为表现受到的影响也相同，但如果提供给测试人员的是安慰剂，则身处高空比身处海平面的行为表现要更差一些。这表明，无论是否摄入酒精，海拔高度可能对心理表现有轻微的影响。</p>
<p>　　国际空间站禁止享用啤酒等有大量泡沫的饮料，可能有另一个原因：没有重力的帮助，液体和气体会在宇航员的胃里不停地翻滚，导致他们不断地打嗝。</p>
<p>　　然而，尽管有严格的规则，这并不意味着太空中的人类不会接触发酵液体。在国际空间站上进行了大量有关酒精的实验—— 但没有发生让众人去饮酒的情况，所以没有人真正了解太空中人体对酒精具体有怎样的反应。</p>
<p>　　NASA发言人斯蒂芬妮?席尔霍尔茨(Stephanie Schierhol）表示：“我们研究了太空中宇航员身体的各种变化，包括微生物层面的。我们有一个营养计划，以确保他们的身体获得保持健康所需要的营养。显然，在实施‘天空实验室(skylab)’项目时，他们曾将雪利酒与宇航员一起送到太空中，但宇航员在零重力飞行时使用雪利酒的测试结果不太好。”天空实验室是美国第一座空间站。</p>
<p>　　然而，尽管有严格的规则，这并不意味着太空中的人类不会接触发酵液体。在国际空间站上进行了大量有关酒精的实验——
但没有发生让众人去饮酒的情况，所以没有人真正了解太空中人体对酒精具体有怎样的反应。</p>
<p>　　NASA发言人斯蒂芬妮?席尔霍尔茨(Stephanie
Schierhol）表示：“我们研究了太空中宇航员身体的各种变化，包括微生物层面的。我们有一个营养计划，以确保他们的身体获得保持健康所需要的营养。显然，在实施‘天空实验室(skylab)’项目时，他们曾将雪利酒与宇航员一起送到太空中，但宇航员在零重力飞行时使用雪利酒的测试结果不太好。”天空实验室是美国第一座空间站。</p>
<p>　　席尔霍尔茨补充说，在测试中使用雪利酒“引发呕吐反射，公众也反对”。</p>
<p>　　也许最令人惊讶的是，人类在月球表面上喝的第一种液体是葡萄酒。前NASA宇航员巴兹·奥尔德林(Buzz Aldrin)在采访和他撰写的书中表示，1969年，在和尼尔·阿姆斯特朗(Neil Armstrong)走出登月舱之前的圣餐仪式上，他喝了少量葡萄酒。举行这一仪式时与地面的通信出现了暂停，因此这一过程从来没有播出。</p>
<p>　　虽然Nasa对太空中酒精的使用有严格的规定，但在这方面俄罗斯过去似乎更为宽松。在其“和平号”空间站上，宇航员允许喝点干邑和伏特加。当他们发现国际空间站将严格禁止饮酒时，显然有不少怨言。</p>
<p>　　然而，奇怪的是，酒仍然能通过各种方式出现在国际空间站上。2015年，日本酿酒商三得利(Suntory)的全球创新中心将该公司一些获奖的威士忌运送到国际空间站，参与一项旨在验证“能否通过利用微重力环境增强酒精饮料醇厚性”的实验。换句话说，在微重力下酒的陈酿过程可能不同，导致它的陈酿进程更快、味道更好。对此，地球上的每家酿酒商都想进一步地了解。</p>
<p>　　几年前，即2011年9月至2014年9月，Nasa赞助了一个试验，研究微重力环境对威士忌中未发酵麦芽与烧焦橡木颗粒的影响，这两种物质能对威士忌的陈酿起帮助作用。在太空中逗留将近1000天后，用于测试的威士忌的单宁成分保持不变——但是太空中橡木颗粒产生了更高浓度的木质素分解产物，这种物质能赋予威士忌特别的风味。</p>
<p>　　Nasa表示：“这种试验不仅对麦芽威士忌行业有影响，而且对整个食品和饮料行业也有影响。送上太空的威士忌与对照样品之间的风味差异是如此显著，需要进一步分析以破解不同口味产生的原因。”</p>
<p>　　也许最令人惊讶的是，人类在月球表面上喝的第一种液体是葡萄酒。前NASA宇航员巴兹·奥尔德林(Buzz
Aldrin)在采访和他撰写的书中表示，1969年，在和尼尔·阿姆斯特朗(Neil
Armstrong)走出登月舱之前的圣餐仪式上，他喝了少量葡萄酒。举行这一仪式时与地面的通信出现了暂停，因此这一过程从来没有播出。</p>
<p>
　　虽然Nasa对太空中酒精的使用有严格的规定，但在这方面俄罗斯过去似乎更为宽松。在其“和平号”空间站上，宇航员允许喝点干邑和伏特加。当他们发现国际空间站将严格禁止饮酒时，显然有不少怨言。</p>
<p>
　　然而，奇怪的是，酒仍然能通过各种方式出现在国际空间站上。2015年，日本酿酒商三得利(Suntory)的全球创新中心将该公司一些获奖的威士忌运送到国际空间站，参与一项旨在验证“能否通过利用微重力环境增强酒精饮料醇厚性”的实验。换句话说，在微重力下酒的陈酿过程可能不同，导致它的陈酿进程更快、味道更好。对此，地球上的每家酿酒商都想进一步地了解。</p>
<p>
　　几年前，即2011年9月至2014年9月，Nasa赞助了一个试验，研究微重力环境对威士忌中未发酵麦芽与烧焦橡木颗粒的影响，这两种物质能对威士忌的陈酿起帮助作用。在太空中逗留将近1000天后，用于测试的威士忌的单宁成分保持不变——但是太空中橡木颗粒产生了更高浓度的木质素分解产物，这种物质能赋予威士忌特别的风味。</p>
<p>
　　Nasa表示：“这种试验不仅对麦芽威士忌行业有影响，而且对整个食品和饮料行业也有影响。送上太空的威士忌与对照样品之间的风味差异是如此显著，需要进一步分析以破解不同口味产生的原因。”</p>
<p>　　因此，即使宇航员自己被禁止在地球轨道上饮酒，但他们正在做的工作可以提高在地上消费的酒的质量。</p>
<p>　　相比之下，执行登陆火星任务的人将远离家乡几年，而不是几个月，因此可能会有人提出有关禁止饮酒的规定可以放松一些。</p>
<p>　　然而，像戴夫?汉森这样的专家认为，继续禁止饮酒并没有什么害处。除了实际的安全问题，饮酒还可能有其它挑战。汉森认为，地球人存在许多社会文化方面的差异，而且人连续几年时间呆在一个狭小的空间里，很容易突然发怒，这些因素都使饮酒问题变得很棘手。</p>
<p><img id="45395153" alt="(Credit: David Frohman/Peachstate Historical Consulting Inc)" src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e592d.jpg" title="宇航员在太空中喝酒会怎么样？后果很严重" /> </p>
<p>
　　然而，像戴夫?汉森这样的专家认为，继续禁止饮酒并没有什么害处。除了实际的安全问题，饮酒还可能有其它挑战。汉森认为，地球人存在许多社会文化方面的差异，而且人连续几年时间呆在一个狭小的空间里，很容易突然发怒，这些因素都使饮酒问题变得很棘手。</p>
<p><img id="45395153" alt="(Credit: David Frohman/Peachstate Historical Consulting Inc)"
src="http://imgtech.gmw.cn/attachement/jpg/site2/20170310/448a5bc1e2861a2c4e592d.jpg"
title="宇航员在太空中喝酒会怎么样？后果很严重"/></p>
<p>
<span face="楷体">　　图注：奥尔德林的圣餐杯回到了地球上</span>
</p>
<p>　　他说：“这是一个政治问题，也是一个文化方面的问题，但不是一个科学上的问题。这将是未来一个可能产生冲突领域，因为人们具有不同的文化背景，他们对饮酒的态度不同。”他进一步指出，如果你与穆斯林、摩门教徒或禁酒主义者分配在同一间宿舍怎么办？面对未来人们可能在一个没有期限的时间内呆在一个有限的空间里，需要“尽早解决”如何协调不同文化观点的问题。</p>
<p><ins></ins>　　所以，当宇航员在地球轨道上时，将还不得不满足于通过欣赏外面的景色来振作精神，而不要指望沉溺于烈酒中。我们留在地球上的人，则可以准备好适量的香槟酒，以迎接他们的归来。</p>
<p>
　　他说：“这是一个政治问题，也是一个文化方面的问题，但不是一个科学上的问题。这将是未来一个可能产生冲突领域，因为人们具有不同的文化背景，他们对饮酒的态度不同。”他进一步指出，如果你与穆斯林、摩门教徒或禁酒主义者分配在同一间宿舍怎么办？面对未来人们可能在一个没有期限的时间内呆在一个有限的空间里，需要“尽早解决”如何协调不同文化观点的问题。</p>
<p>
<ins></ins>
　　所以，当宇航员在地球轨道上时，将还不得不满足于通过欣赏外面的景色来振作精神，而不要指望沉溺于烈酒中。我们留在地球上的人，则可以准备好适量的香槟酒，以迎接他们的归来。
</p>
<p>　　原标题：他晚于阿姆斯特朗登月 却是首个敢在月球喝酒的人</p>
<p><strong>　　出品︱网易科学人栏目组 胖胖</strong></p>
<p><strong>　　作者︱春春</strong>
<a href="http://www.gmw.cn/" target="_blank"><img src="https://img.gmw.cn/pic/content_logo.png" title="返回光明网首页" /></a>
<a href="http://www.gmw.cn/" target="_blank"><img
src="https://img.gmw.cn/pic/content_logo.png" title="返回光明网首页"/></a>
</p>
<p>[责任编辑:肖春芳]</p>
</div>

2135
app/src/main/assets/readability/test/test-pages/gmw/source.html
View File

File diff suppressed because it is too large Load Diff

667
app/src/main/assets/readability/test/test-pages/google-sre-book-1/expected.html
View File

@ -1,49 +1,93 @@
<div id="readability-page-1" class="page">
<section data-type="chapter" id="maia-main" role="main">
<h2> Monitoring Distributed Systems </h2>
<p> Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page. </p>
<p> Google’s SRE teams have some basic principles and best practices for building successful
monitoring and alerting systems. This chapter offers guidelines for what issues should
interrupt a human via a page, and how to deal with issues that aren’t serious enough to
trigger a page. </p>
<section data-type="sect1" id="definitions-2ksZhN">
<h2> Definitions </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="terminology" id="id-DnC1SWFMhD"></a>There’s no uniformly shared vocabulary for discussing all topics related to monitoring. Even within Google, usage of the following terms varies, but the most common interpretations are listed here.
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="terminology" id="id-DnC1SWFMhD"></a>There’s no uniformly shared
vocabulary for discussing all topics related to monitoring. Even within Google,
usage of the following terms varies, but the most common interpretations are listed
here.
</p>
<dl>
<dd>
<p> Collecting, processing, aggregating, and displaying real-time quantitative data about a system, such as query counts and types, error counts and types, processing times, and server lifetimes. </p>
<p> Collecting, processing, aggregating, and displaying real-time quantitative
data about a system, such as query counts and types, error counts and types,
processing times, and server lifetimes. </p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="white-box monitoring" id="id-9nCjSDS4tZILhX"></a>Monitoring based on metrics exposed by the internals of the system, including logs, interfaces like the Java Virtual Machine Profiling Interface, or an HTTP handler that emits internal statistics.
<a data-type="indexterm" data-primary="white-box monitoring"
id="id-9nCjSDS4tZILhX"></a>Monitoring based on metrics exposed by the
internals of the system, including logs, interfaces like the Java Virtual
Machine Profiling Interface, or an HTTP handler that emits internal
statistics.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCxSrSgTWIdhb"></a>Testing externally visible behavior as a user would see it.
<a data-type="indexterm" data-primary="black-box monitoring"
id="id-zdCxSrSgTWIdhb"></a>Testing externally visible behavior as a user
would see it.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="dashboards" data-secondary="defined" id="id-VMCPS2SribIkh4"></a>An application (usually web-based) that provides a summary view of a service’s core metrics. A dashboard may have filters, selectors, and so on, but is prebuilt to expose the metrics most important to its users. The dashboard might also display team information such as ticket queue length, a list of high-priority bugs, the current on-call engineer for a given area of responsibility, or recent pushes.
<a data-type="indexterm" data-primary="dashboards" data-secondary="defined"
id="id-VMCPS2SribIkh4"></a>An application (usually web-based) that
provides a summary view of a service’s core metrics. A dashboard may have
filters, selectors, and so on, but is prebuilt to expose the metrics most
important to its users. The dashboard might also display team information
such as ticket queue length, a list of high-priority bugs, the current
on-call engineer for a given area of responsibility, or recent pushes.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="alerts" data-secondary="defined" id="id-wqC7SvSPUAIVhQ"></a>A notification intended to be read by a human and that is pushed to a system such as a bug or ticket queue, an email alias, or a pager. Respectively, these alerts are classified as <em>tickets</em>, <em>email alerts</em>,<sup><a data-type="noteref" id="id-LvQuvtYS7UvI8h4-marker" href="#id-LvQuvtYS7UvI8h4">22</a></sup> and <em>pages</em>.
<a data-type="indexterm" data-primary="alerts" data-secondary="defined"
id="id-wqC7SvSPUAIVhQ"></a>A notification intended to be read by a human
and that is pushed to a system such as a bug or ticket queue, an email
alias, or a pager. Respectively, these alerts are classified as
<em>tickets</em>, <em>email alerts</em>,<sup><a data-type="noteref"
id="id-LvQuvtYS7UvI8h4-marker"
href="#id-LvQuvtYS7UvI8h4">22</a></sup>
and <em>pages</em>.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="root cause" data-secondary="defined" id="id-PnCpSaSKsgIjho"></a>A defect in a software or human system that, if repaired, instills confidence that this event won’t happen again in the same way. A given incident might have multiple root causes: for example, perhaps it was caused by a combination of insufficient process automation, software that crashed on bogus input, <em>and</em> insufficient testing of the script used to generate the configuration. Each of these factors might stand alone as a root cause, and each should be repaired.
<a data-type="indexterm" data-primary="root cause" data-secondary="defined"
id="id-PnCpSaSKsgIjho"></a>A defect in a software or human system that,
if repaired, instills confidence that this event won’t happen again in the
same way. A given incident might have multiple root causes: for example,
perhaps it was caused by a combination of insufficient process automation,
software that crashed on bogus input, <em>and</em> insufficient testing of
the script used to generate the configuration. Each of these factors might
stand alone as a root cause, and each should be repaired.
</p>
</dd>
<dt id="node-and-machine"> Node and machine </dt>
<dt id="node-and-machine"> Node and machine</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="machines" data-secondary="defined" id="id-XmC9SkSlfnI1hK"></a>Used interchangeably to indicate a single instance of a running kernel in either a physical server, virtual machine, or container. There might be multiple <em>services</em> worth monitoring on a single machine. The services may either be:
<a data-type="indexterm" data-primary="machines" data-secondary="defined"
id="id-XmC9SkSlfnI1hK"></a>Used interchangeably to indicate a single
instance of a running kernel in either a physical server, virtual machine,
or container. There might be multiple <em>services</em> worth monitoring on
a single machine. The services may either be:
</p>
<ul>
<li>Related to each other: for example, a caching server and a web server </li>
<li>Unrelated services sharing hardware: for example, a code repository and a master for a configuration system like <a href="https://puppetlabs.com/puppet/puppet-open-source" target="_blank">Puppet</a> or <a href="https://www.chef.io/chef/" target="_blank">Chef</a>
<li>Related to each other: for example, a caching server and a web server
</li>
<li>Unrelated services sharing hardware: for example, a code repository and
a master for a configuration system like <a
href="https://puppetlabs.com/puppet/puppet-open-source"
target="_blank">Puppet</a> or <a
href="https://www.chef.io/chef/" target="_blank">Chef</a>
</li>
</ul>
</dd>
@ -55,253 +99,600 @@
<section data-type="sect1" id="why-monitor-pWsBTZ">
<h2> Why Monitor? </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="benefits of monitoring" id="id-kVCkSpFnTl"></a>There are many reasons to monitor a system, including:
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="benefits of monitoring" id="id-kVCkSpFnTl"></a>There are many
reasons to monitor a system, including:
</p>
<dl>
<dd>
<p> How big is my database and how fast is it growing? How quickly is my daily-active user count growing? </p>
<p> How big is my database and how fast is it growing? How quickly is my
daily-active user count growing? </p>
</dd>
<dd>
<p> Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How much better is my memcache hit rate with an extra node? Is my site slower than it was last week? </p>
<p> Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How
much better is my memcache hit rate with an extra node? Is my site slower
than it was last week? </p>
</dd>
<dd>
<p> Something is broken, and somebody needs to fix it right now! Or, something might break soon, so somebody should look soon. </p>
<p> Something is broken, and somebody needs to fix it right now! Or, something
might break soon, so somebody should look soon. </p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="dashboards" data-secondary="benefits of" id="id-rjCXSOS0iDIGT8"></a>Dashboards should answer basic questions about your service, and normally include some form of the four golden signals (discussed in <a data-type="xref" href="#xref_monitoring_golden-signals">The Four Golden Signals</a>).
<a data-type="indexterm" data-primary="dashboards"
data-secondary="benefits of" id="id-rjCXSOS0iDIGT8"></a>Dashboards should
answer basic questions about your service, and normally include some form of
the four golden signals (discussed in <a data-type="xref"
href="#xref_monitoring_golden-signals">The
Four Golden Signals</a>).
</p>
</dd>
<dd>
<p> Our latency just shot up; what else happened around the same time? </p>
</dd>
</dl>
<p> System monitoring is also helpful in supplying raw input into business analytics and in facilitating analysis of security breaches. Because this book focuses on the engineering domains in which SRE has particular expertise, we won’t discuss these applications of monitoring here. </p>
<p> Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to tell us what’s about to break. When the system isn’t able to automatically fix itself, we want a human to investigate the alert, determine if there’s a real problem at hand, mitigate the problem, and determine the root cause of the problem. Unless you’re performing security auditing on very narrowly scoped components of a system, you should never trigger an alert simply because "something seems a bit weird." </p>
<p> Paging a human is a quite expensive use of an employee’s time. If an employee is at work, a page interrupts their workflow. If the employee is at home, a page interrupts their personal time, and perhaps even their sleep. When pages occur too frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes even ignoring a "real" page that’s masked by the noise. Outages can be prolonged because other noise interferes with a rapid diagnosis and fix. Effective alerting systems have good signal and very low noise. </p>
<p> System monitoring is also helpful in supplying raw input into business analytics and
in facilitating analysis of security breaches. Because this book focuses on the
engineering domains in which SRE has particular expertise, we won’t discuss these
applications of monitoring here. </p>
<p> Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to
tell us what’s about to break. When the system isn’t able to automatically fix
itself, we want a human to investigate the alert, determine if there’s a real
problem at hand, mitigate the problem, and determine the root cause of the problem.
Unless you’re performing security auditing on very narrowly scoped components of a
system, you should never trigger an alert simply because "something seems a bit
weird." </p>
<p> Paging a human is a quite expensive use of an employee’s time. If an employee is at
work, a page interrupts their workflow. If the employee is at home, a page
interrupts their personal time, and perhaps even their sleep. When pages occur too
frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes
even ignoring a "real" page that’s masked by the noise. Outages can be prolonged
because other noise interferes with a rapid diagnosis and fix. Effective alerting
systems have good signal and very low noise. </p>
</section>
<section data-type="sect1" id="setting-reasonable-expectations-for-monitoring-o8svcM">
<h2> Setting Reasonable Expectations for Monitoring </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="setting expectations for" id="id-4nCqSYFQcE"></a>Monitoring a complex application is a significant engineering endeavor in and of itself. Even with substantial existing infrastructure for instrumentation, collection, display, and alerting in place, a Google SRE team with 10–12 members typically has one or sometimes two members whose primary assignment is to build and maintain monitoring systems for their service. This number has decreased over time as we generalize and centralize common monitoring infrastructure, but every SRE team typically has at least one “monitoring person.” (That being said, while it can be fun to have access to traffic graph dashboards and the like, SRE teams carefully avoid any situation that requires someone to “stare at a screen to watch for problems.”)
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="setting expectations for" id="id-4nCqSYFQcE"></a>Monitoring a
complex application is a significant engineering endeavor in and of itself. Even
with substantial existing infrastructure for instrumentation, collection, display,
and alerting in place, a Google SRE team with 10–12 members typically has one or
sometimes two members whose primary assignment is to build and maintain monitoring
systems for their service. This number has decreased over time as we generalize and
centralize common monitoring infrastructure, but every SRE team typically has at
least one “monitoring person.” (That being said, while it can be fun to have access
to traffic graph dashboards and the like, SRE teams carefully avoid any situation
that requires someone to “stare at a screen to watch for problems.”)
</p>
<p>
<a data-type="indexterm" data-primary="post hoc analysis" id="id-JnCDSjIVcG"></a>In general, Google has trended toward simpler and faster monitoring systems, with better tools for <em>post hoc</em> analysis. We avoid "magic" systems that try to learn thresholds or automatically detect causality. Rules that detect unexpected changes in end-user request rates are one counterexample; while these rules are still kept as simple as possible, they give a very quick detection of a very simple, specific, severe anomaly. Other uses of monitoring data such as capacity planning and traffic prediction can tolerate more fragility, and thus, more complexity. Observational experiments conducted over a very long time horizon (months or years) with a low sampling rate (hours or days) can also often tolerate more fragility because occasional missed samples won’t hide a long-running trend.
<a data-type="indexterm" data-primary="post hoc analysis" id="id-JnCDSjIVcG"></a>In
general, Google has trended toward simpler and faster monitoring systems, with
better tools for <em>post hoc</em> analysis. We avoid "magic" systems that try to
learn thresholds or automatically detect causality. Rules that detect unexpected
changes in end-user request rates are one counterexample; while these rules are
still kept as simple as possible, they give a very quick detection of a very simple,
specific, severe anomaly. Other uses of monitoring data such as capacity planning
and traffic prediction can tolerate more fragility, and thus, more complexity.
Observational experiments conducted over a very long time horizon (months or years)
with a low sampling rate (hours or days) can also often tolerate more fragility
because occasional missed samples won’t hide a long-running trend.
</p>
<p>
<a data-type="indexterm" data-primary="dependency hierarchies" id="id-9nCjSOtmcj"></a>Google SRE has experienced only limited success with complex dependency hierarchies. We seldom use rules such as, "If I know the database is slow, alert for a slow database; otherwise, alert for the website being generally slow." Dependency-reliant rules usually pertain to very stable parts of our system, such as our system for draining user traffic away from a datacenter. For example, "If a datacenter is drained, then don’t alert me on its latency" is one common datacenter alerting rule. Few teams at Google maintain complex dependency hierarchies because our infrastructure has a steady rate of continuous refactoring.
<a data-type="indexterm" data-primary="dependency hierarchies"
id="id-9nCjSOtmcj"></a>Google SRE has experienced only limited success with
complex dependency hierarchies. We seldom use rules such as, "If I know the database
is slow, alert for a slow database; otherwise, alert for the website being generally
slow." Dependency-reliant rules usually pertain to very stable parts of our system,
such as our system for draining user traffic away from a datacenter. For example,
"If a datacenter is drained, then don’t alert me on its latency" is one common
datacenter alerting rule. Few teams at Google maintain complex dependency
hierarchies because our infrastructure has a steady rate of continuous refactoring.
</p>
<p> Some of the ideas described in this chapter are still aspirational: there is always room to move more rapidly from symptom to root cause(s), especially in ever-changing systems. So while this chapter sets out some goals for monitoring systems, and some ways to achieve these goals, it’s important that monitoring systems—especially the critical path from the onset of a production problem, through a page to a human, through basic triage and deep debugging—be kept simple and comprehensible by everyone on the team. </p>
<p> Similarly, to keep noise low and signal high, the elements of your monitoring system that direct to a pager need to be very simple and robust. Rules that generate alerts for humans should be simple to understand and represent a clear failure. </p>
<p> Some of the ideas described in this chapter are still aspirational: there is always
room to move more rapidly from symptom to root cause(s), especially in ever-changing
systems. So while this chapter sets out some goals for monitoring systems, and some
ways to achieve these goals, it’s important that monitoring systems—especially the
critical path from the onset of a production problem, through a page to a human,
through basic triage and deep debugging—be kept simple and comprehensible by
everyone on the team. </p>
<p> Similarly, to keep noise low and signal high, the elements of your monitoring system
that direct to a pager need to be very simple and robust. Rules that generate alerts
for humans should be simple to understand and represent a clear failure. </p>
</section>
<section data-type="sect1" id="symptoms-versus-causes-g0sEi4">
<h2> Symptoms Versus Causes </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="symptoms vs. causes" id="id-JnCDSlFmiG"></a>Your monitoring system should address two questions: what’s broken, and why?
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="symptoms vs. causes" id="id-JnCDSlFmiG"></a>Your monitoring
system should address two questions: what’s broken, and why?
</p>
<p> The "what’s broken" indicates the symptom; the "why" indicates a (possibly intermediate) cause. <a data-type="xref" href="#table_monitoring_symptoms">Table 6-1</a> lists some hypothetical symptoms and corresponding causes. </p>
<p> The "what’s broken" indicates the symptom; the "why" indicates a (possibly
intermediate) cause. <a data-type="xref" href="#table_monitoring_symptoms">Table
6-1</a> lists some hypothetical symptoms and corresponding causes. </p>
<table id="table_monitoring_symptoms">
<caption>
<span>Table 6-1.</span> Example symptoms and causes
</caption>
<thead>
<tr>
<th>
<strong>Symptom</strong>
</th>
<th>
<strong>Cause</strong>
</th>
</tr>
<tr>
<th>
<strong>Symptom</strong>
</th>
<th>
<strong>Cause</strong>
</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<p>
<strong>I’m serving HTTP 500s or 404s</strong>
</p>
</td>
<td>
<p> Database servers are refusing connections </p>
</td>
</tr>
<tr>
<td>
<p>
<strong>My responses are slow</strong>
</p>
</td>
<td>
<p> CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under a rack, visible as partial packet loss </p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Users in Antarctica aren’t receiving animated cat GIFs</strong>
</p>
</td>
<td>
<p> Your Content Distribution Network hates scientists and felines, and thus blacklisted some client IPs </p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Private content is world-readable</strong>
</p>
</td>
<td>
<p> A new software push caused ACLs to be forgotten and allowed all requests </p>
</td>
</tr>
<tr>
<td>
<p>
<strong>I’m serving HTTP 500s or 404s</strong>
</p>
</td>
<td>
<p> Database servers are refusing connections </p>
</td>
</tr>
<tr>
<td>
<p>
<strong>My responses are slow</strong>
</p>
</td>
<td>
<p> CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under
a rack, visible as partial packet loss </p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Users in Antarctica aren’t receiving animated cat GIFs</strong>
</p>
</td>
<td>
<p> Your Content Distribution Network hates scientists and felines, and thus
blacklisted some client IPs </p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Private content is world-readable</strong>
</p>
</td>
<td>
<p> A new software push caused ACLs to be forgotten and allowed all
requests </p>
</td>
</tr>
</tbody>
</table>
<p> "What" versus "why" is one of the most important distinctions in writing good monitoring with maximum signal and minimum noise. </p>
<p> "What" versus "why" is one of the most important distinctions in writing good
monitoring with maximum signal and minimum noise. </p>
</section>
<section data-type="sect1" id="black-box-versus-white-box-q8sJuw">
<h2> Black-Box Versus White-Box </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="blackbox vs. whitebox" id="id-9nCjSvFVuj"></a><a data-type="indexterm" data-primary="white-box monitoring" id="id-ZbC1FMFEu7"></a><a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCXIGFvuy"></a>We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring. The simplest way to think about black-box monitoring versus white-box monitoring is that black-box monitoring is symptom-oriented and represents active—not predicted—problems: "The system isn’t working correctly, right now." White-box monitoring depends on the ability to inspect the innards of the system, such as logs or HTTP endpoints, with instrumentation. White-box monitoring therefore allows detection of imminent problems, failures masked by retries, and so forth.
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="blackbox vs. whitebox" id="id-9nCjSvFVuj"></a><a
data-type="indexterm" data-primary="white-box monitoring"
id="id-ZbC1FMFEu7"></a><a data-type="indexterm"
data-primary="black-box monitoring"
id="id-zdCXIGFvuy"></a>We combine heavy use of
white-box monitoring with modest but critical uses of black-box monitoring. The
simplest way to think about black-box monitoring versus white-box monitoring is that
black-box monitoring is symptom-oriented and represents active—not
predicted—problems: "The system isn’t working correctly, right now." White-box
monitoring depends on the ability to inspect the innards of the system, such as logs
or HTTP endpoints, with instrumentation. White-box monitoring therefore allows
detection of imminent problems, failures masked by retries, and so forth.
</p>
<p> Note that in a multilayered system, one person’s symptom is another person’s cause. For example, suppose that a database’s performance is slow. Slow database reads are a symptom for the database SRE who detects them. However, for the frontend SRE observing a slow website, the same slow database reads are a cause. Therefore, white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented, depending on just how informative your white-box is. </p>
<p> When collecting telemetry for debugging, white-box monitoring is essential. If web servers seem slow on database-heavy requests, you need to know both how fast the web server perceives the database to be, and how fast the database believes itself to be. Otherwise, you can’t distinguish an actually slow database server from a network problem between your web server and your database. </p>
<p> For paging, black-box monitoring has the key benefit of forcing discipline to only nag a human when a problem is both already ongoing and contributing to real symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box monitoring is fairly useless. </p>
<p> Note that in a multilayered system, one person’s symptom is another person’s cause.
For example, suppose that a database’s performance is slow. Slow database reads are
a symptom for the database SRE who detects them. However, for the frontend SRE
observing a slow website, the same slow database reads are a cause. Therefore,
white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented,
depending on just how informative your white-box is. </p>
<p> When collecting telemetry for debugging, white-box monitoring is essential. If web
servers seem slow on database-heavy requests, you need to know both how fast the web
server perceives the database to be, and how fast the database believes itself to
be. Otherwise, you can’t distinguish an actually slow database server from a network
problem between your web server and your database. </p>
<p> For paging, black-box monitoring has the key benefit of forcing discipline to only
nag a human when a problem is both already ongoing and contributing to real
symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box
monitoring is fairly useless. </p>
</section>
<section data-type="sect1" id="xref_monitoring_golden-signals">
<h2> The Four Golden Signals </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="four golden signals of" id="id-ZbCxSMFjU7"></a>The four golden signals of monitoring are latency, traffic, errors, and saturation. If you can only measure four metrics of your user-facing system, focus on these four.
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="four golden signals of" id="id-ZbCxSMFjU7"></a>The four golden
signals of monitoring are latency, traffic, errors, and saturation. If you can only
measure four metrics of your user-facing system, focus on these four.
</p>
<dl>
<dd>
<p>
<a data-type="indexterm" data-primary="service latency" data-secondary="monitoring for" id="id-yYCASJS9FKIWUb"></a><a data-type="indexterm" data-primary="latency" data-secondary="monitoring for" id="id-VMCpF2SXFbIwU4"></a><a data-type="indexterm" data-primary="request latency" id="id-rjCeIOSKFDIaU8"></a><a data-type="indexterm" data-primary="user requests" data-secondary="request latency monitoring" id="id-wqCDtvSGFAIMUQ"></a>The time it takes to service a request. It’s important to distinguish between the latency of successful requests and the latency of failed requests. For example, an HTTP 500 error triggered due to loss of connection to a database or other critical backend might be served very quickly; however, as an HTTP 500 error indicates a failed request, factoring 500s into your overall latency might result in misleading calculations. On the other hand, a slow error is even worse than a fast error! Therefore, it’s important to track error latency, as opposed to just filtering out errors.
<a data-type="indexterm" data-primary="service latency"
data-secondary="monitoring for" id="id-yYCASJS9FKIWUb"></a><a
data-type="indexterm" data-primary="latency"
data-secondary="monitoring for" id="id-VMCpF2SXFbIwU4"></a><a
data-type="indexterm" data-primary="request latency"
id="id-rjCeIOSKFDIaU8"></a><a data-type="indexterm"
data-primary="user requests"
data-secondary="request latency monitoring"
id="id-wqCDtvSGFAIMUQ"></a>The time it
takes to service a request. It’s important to distinguish between the
latency of successful requests and the latency of failed requests. For
example, an HTTP 500 error triggered due to loss of connection to a database
or other critical backend might be served very quickly; however, as an HTTP
500 error indicates a failed request, factoring 500s into your overall
latency might result in misleading calculations. On the other hand, a slow
error is even worse than a fast error! Therefore, it’s important to track
error latency, as opposed to just filtering out errors.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="user requests" data-secondary="traffic analysis" id="id-rjCXSOSxtDIaU8"></a><a data-type="indexterm" data-primary="traffic analysis" id="id-wqC4FvSBtAIMUQ"></a>A measure of how much demand is being placed on your system, measured in a high-level system-specific metric. For a web service, this measurement is usually HTTP requests per second, perhaps broken out by the nature of the requests (e.g., static versus dynamic content). For an audio streaming system, this measurement might focus on network I/O rate or concurrent sessions. For a key-value storage system, this measurement might be transactions and retrievals per <span>second</span>.
<a data-type="indexterm" data-primary="user requests"
data-secondary="traffic analysis" id="id-rjCXSOSxtDIaU8"></a><a
data-type="indexterm" data-primary="traffic analysis"
id="id-wqC4FvSBtAIMUQ"></a>A measure of how much demand is being placed
on your system, measured in a high-level system-specific metric. For a web
service, this measurement is usually HTTP requests per second, perhaps
broken out by the nature of the requests (e.g., static versus dynamic
content). For an audio streaming system, this measurement might focus on
network I/O rate or concurrent sessions. For a key-value storage system,
this measurement might be transactions and retrievals per
<span>second</span>.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="error rates" id="id-x1C4SjSlTLIMUJ"></a><a data-type="indexterm" data-primary="user requests" data-secondary="monitoring failures" id="id-PnCxFaS0TgIVUo"></a>The rate of requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for example, an HTTP 200 success response, but coupled with the wrong content), or by policy (for example, "If you committed to one-second response times, any request over one second is an error"). Where protocol response codes are insufficient to express all failure conditions, secondary (internal) protocols may be necessary to track partial failure modes. Monitoring these cases can be drastically different: catching HTTP 500s at your load balancer can do a decent job of catching all completely failed requests, while only end-to-end system tests can detect that you’re serving the wrong content.
<a data-type="indexterm" data-primary="error rates"
id="id-x1C4SjSlTLIMUJ"></a><a data-type="indexterm"
data-primary="user requests"
data-secondary="monitoring failures"
id="id-PnCxFaS0TgIVUo"></a>The rate of
requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for
example, an HTTP 200 success response, but coupled with the wrong content),
or by policy (for example, "If you committed to one-second response times,
any request over one second is an error"). Where protocol response codes are
insufficient to express all failure conditions, secondary (internal)
protocols may be necessary to track partial failure modes. Monitoring these
cases can be drastically different: catching HTTP 500s at your load balancer
can do a decent job of catching all completely failed requests, while only
end-to-end system tests can detect that you’re serving the wrong content.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="saturation" id="id-OnCNS2S4iDIYU8"></a>How "full" your service is. A measure of your system fraction, emphasizing the resources that are most constrained (e.g., in a memory-constrained system, show memory; in an I/O-constrained system, show I/O). Note that many systems degrade in performance before they achieve 100% utilization, so having a utilization target is essential.
<a data-type="indexterm" data-primary="saturation"
id="id-OnCNS2S4iDIYU8"></a>How "full" your service is. A measure of your
system fraction, emphasizing the resources that are most constrained (e.g.,
in a memory-constrained system, show memory; in an I/O-constrained system,
show I/O). Note that many systems degrade in performance before they achieve
100% utilization, so having a utilization target is essential.
</p>
<p> In complex systems, saturation can be supplemented with higher-level load measurement: can your service properly handle double the traffic, handle only 10% more traffic, or handle even less traffic than it currently receives? For very simple services that have no parameters that alter the complexity of the request (e.g., "Give me a nonce" or "I need a globally unique monotonic integer") that rarely change configuration, a static value from a load test might be adequate. As discussed in the previous paragraph, however, most services need to use indirect signals like CPU utilization or network bandwidth that have a known upper bound. Latency increases are often a leading indicator of saturation. Measuring your 99th percentile response time over some small window (e.g., one minute) can give a very early signal of saturation. </p>
<p> Finally, saturation is also concerned with predictions of impending saturation, such as "It looks like your database will fill its hard drive in 4 hours." </p>
<p> In complex systems, saturation can be supplemented with higher-level load
measurement: can your service properly handle double the traffic, handle
only 10% more traffic, or handle even less traffic than it currently
receives? For very simple services that have no parameters that alter the
complexity of the request (e.g., "Give me a nonce" or "I need a globally
unique monotonic integer") that rarely change configuration, a static value
from a load test might be adequate. As discussed in the previous paragraph,
however, most services need to use indirect signals like CPU utilization or
network bandwidth that have a known upper bound. Latency increases are often
a leading indicator of saturation. Measuring your 99th percentile response
time over some small window (e.g., one minute) can give a very early signal
of saturation. </p>
<p> Finally, saturation is also concerned with predictions of impending
saturation, such as "It looks like your database will fill its hard drive in
4 hours." </p>
</dd>
</dl>
<p> If you measure all four golden signals and page a human when one signal is problematic (or, in the case of saturation, nearly problematic), your service will be at least decently covered by monitoring. </p>
<p> If you measure all four golden signals and page a human when one signal is
problematic (or, in the case of saturation, nearly problematic), your service will
be at least decently covered by monitoring. </p>
</section>
<section data-type="sect1" id="worrying-about-your-tail-or-instrumentation-and-performance-Yms9Ck">
<section data-type="sect1"
id="worrying-about-your-tail-or-instrumentation-and-performance-Yms9Ck">
<h2> Worrying About Your Tail (or, Instrumentation and Performance) </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="instrumentation and performance" id="id-zdCxSGFQCy"></a><a data-type="indexterm" data-primary="performance" data-secondary="monitoring" id="id-yYCyFpFdCr"></a>When building a monitoring system from scratch, it’s tempting to design a system based upon the mean of some quantity: the mean latency, the mean CPU usage of your nodes, or the mean fullness of your databases. The danger presented by the latter two cases is obvious: CPUs and databases can easily be utilized in a very imbalanced way. The same holds for latency. If you run a web service with an average latency of 100 ms at 1,000 requests per second, 1% of requests might easily take 5 seconds.<sup><a data-type="noteref" id="id-QQLuAIXFxCz-marker" href="#id-QQLuAIXFxCz">23</a></sup> If your users depend on several such web services to render their page, the 99th percentile of one backend can easily become the median response of your <span>frontend</span>.
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="instrumentation and performance" id="id-zdCxSGFQCy"></a><a
data-type="indexterm" data-primary="performance" data-secondary="monitoring"
id="id-yYCyFpFdCr"></a>When building a monitoring system from scratch, it’s
tempting to design a system based upon the mean of some quantity: the mean latency,
the mean CPU usage of your nodes, or the mean fullness of your databases. The danger
presented by the latter two cases is obvious: CPUs and databases can easily be
utilized in a very imbalanced way. The same holds for latency. If you run a web
service with an average latency of 100 ms at 1,000 requests per second, 1% of
requests might easily take 5 seconds.<sup><a data-type="noteref"
id="id-QQLuAIXFxCz-marker"
href="#id-QQLuAIXFxCz">23</a></sup> If
your users depend on several such web services to render their page, the 99th
percentile of one backend can easily become the median response of your <span>frontend</span>.
</p>
<p> The simplest way to differentiate between a slow average and a very slow "tail" of requests is to collect request counts bucketed by latencies (suitable for rendering a histogram), rather than actual latencies: how many requests did I serve that took between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between 100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately exponentially (in this case by factors of roughly 3) is often an easy way to visualize the distribution of your requests. </p>
<p> The simplest way to differentiate between a slow average and a very slow "tail" of
requests is to collect request counts bucketed by latencies (suitable for rendering
a histogram), rather than actual latencies: how many requests did I serve that took
between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between
100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately
exponentially (in this case by factors of roughly 3) is often an easy way to
visualize the distribution of your requests. </p>
</section>
<section data-type="sect1" id="choosing-an-appropriate-resolution-for-measurements-vJsBsE">
<h2> Choosing an Appropriate Resolution for Measurements </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="resolution" id="id-yYCASpFxsr"></a>Different aspects of a system should be measured with different levels of granularity. For example:
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="resolution" id="id-yYCASpFxsr"></a>Different aspects of a system
should be measured with different levels of granularity. For example:
</p>
<ul>
<li>Observing CPU load over the time span of a minute won’t reveal even quite long-lived spikes that drive high tail latencies. </li>
<li>On the other hand, for a web service targeting no more than 9 hours aggregate downtime per year (99.9% annual uptime), probing for a 200 (success) status more than once or twice a minute is probably unnecessarily frequent. </li>
<li>Similarly, checking hard drive fullness for a service targeting 99.9% availability more than once every 1–2 minutes is probably unnecessary. </li>
<li>Observing CPU load over the time span of a minute won’t reveal even quite
long-lived spikes that drive high tail latencies.
</li>
<li>On the other hand, for a web service targeting no more than 9 hours aggregate
downtime per year (99.9% annual uptime), probing for a 200 (success) status more
than once or twice a minute is probably unnecessarily frequent.
</li>
<li>Similarly, checking hard drive fullness for a service targeting 99.9%
availability more than once every 1–2 minutes is probably unnecessary.
</li>
</ul>
<p> Take care in how you structure the granularity of your measurements. Collecting per-second measurements of CPU load might yield interesting data, but such frequent measurements may be very expensive to collect, store, and analyze. If your monitoring goal calls for high resolution but doesn’t require extremely low latency, you can reduce these costs by performing internal sampling on the server, then configuring an external system to collect and aggregate that distribution over time or across servers. You might: </p>
<p> Take care in how you structure the granularity of your measurements. Collecting
per-second measurements of CPU load might yield interesting data, but such frequent
measurements may be very expensive to collect, store, and analyze. If your
monitoring goal calls for high resolution but doesn’t require extremely low latency,
you can reduce these costs by performing internal sampling on the server, then
configuring an external system to collect and aggregate that distribution over time
or across servers. You might: </p>
<ol>
<li>Record the current CPU utilization each second. </li>
<li>Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second. </li>
<li>Aggregate those values every minute. </li>
<li>Record the current CPU utilization each second.</li>
<li>Using buckets of 5% granularity, increment the appropriate CPU utilization
bucket each second.
</li>
<li>Aggregate those values every minute.</li>
</ol>
<p> This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention. </p>
<p> This strategy allows you to observe brief CPU hotspots without incurring very high
cost due to collection and retention. </p>
</section>
<section data-type="sect1" id="as-simple-as-possible-no-simpler-lqskHx">
<h2> As Simple as Possible, No Simpler </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="avoiding complexity in" id="id-VMCPSrFpHm"></a>Piling all these requirements on top of each other can add up to a very complex monitoring system—your system might end up with the following levels of complexity:
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="avoiding complexity in" id="id-VMCPSrFpHm"></a>Piling all these
requirements on top of each other can add up to a very complex monitoring
system—your system might end up with the following levels of complexity:
</p>
<ul>
<li>Alerts on different latency thresholds, at different percentiles, on all kinds of different metrics </li>
<li>Extra code to detect and expose possible causes </li>
<li>Associated dashboards for each of these possible causes </li>
<li>Alerts on different latency thresholds, at different percentiles, on all kinds
of different metrics
</li>
<li>Extra code to detect and expose possible causes</li>
<li>Associated dashboards for each of these possible causes</li>
</ul>
<p> The sources of potential complexity are never-ending. Like all software systems, monitoring can become so complex that it’s fragile, complicated to change, and a maintenance burden. </p>
<p> Therefore, design your monitoring system with an eye toward simplicity. In choosing what to monitor, keep the following guidelines in mind: </p>
<p> The sources of potential complexity are never-ending. Like all software systems,
monitoring can become so complex that it’s fragile, complicated to change, and a
maintenance burden. </p>
<p> Therefore, design your monitoring system with an eye toward simplicity. In choosing
what to monitor, keep the following guidelines in mind: </p>
<ul>
<li>The rules that catch real incidents most often should be as simple, predictable, and reliable as possible. </li>
<li>Data collection, aggregation, and alerting configuration that is rarely exercised (e.g., less than once a quarter for some SRE teams) should be up for removal. </li>
<li>Signals that are collected, but not exposed in any prebaked dashboard nor used by any alert, are candidates for removal. </li>
<li>The rules that catch real incidents most often should be as simple, predictable,
and reliable as possible.
</li>
<li>Data collection, aggregation, and alerting configuration that is rarely
exercised (e.g., less than once a quarter for some SRE teams) should be up for
removal.
</li>
<li>Signals that are collected, but not exposed in any prebaked dashboard nor used
by any alert, are candidates for removal.
</li>
</ul>
<p> In Google’s experience, basic collection and aggregation of metrics, paired with alerting and dashboards, has worked well as a relatively standalone system. (In fact Google’s monitoring system is broken up into several binaries, but typically people learn about all aspects of these binaries.) It can be tempting to combine monitoring with other aspects of inspecting complex systems, such as detailed system profiling, single-process debugging, tracking details about exceptions or crashes, load testing, log collection and analysis, or traffic inspection. While most of these subjects share commonalities with basic monitoring, blending together too many results in overly complex and fragile systems. As in many other aspects of software engineering, maintaining distinct systems with clear, simple, loosely coupled points of integration is a better strategy (for example, using web APIs for pulling summary data in a format that can remain constant over an extended period of time). </p>
<p> In Google’s experience, basic collection and aggregation of metrics, paired with
alerting and dashboards, has worked well as a relatively standalone system. (In fact
Google’s monitoring system is broken up into several binaries, but typically people
learn about all aspects of these binaries.) It can be tempting to combine monitoring
with other aspects of inspecting complex systems, such as detailed system profiling,
single-process debugging, tracking details about exceptions or crashes, load
testing, log collection and analysis, or traffic inspection. While most of these
subjects share commonalities with basic monitoring, blending together too many
results in overly complex and fragile systems. As in many other aspects of software
engineering, maintaining distinct systems with clear, simple, loosely coupled points
of integration is a better strategy (for example, using web APIs for pulling summary
data in a format that can remain constant over an extended period of time). </p>
</section>
<section data-type="sect1" id="tying-these-principles-together-nqsJfw">
<h2> Tying These Principles Together </h2>
<p> The principles discussed in this chapter can be tied together into a philosophy on monitoring and alerting that’s widely endorsed and followed within Google SRE teams. While this monitoring philosophy is a bit aspirational, it’s a good starting point for writing or reviewing a new alert, and it can help your organization ask the right questions, regardless of the size of your organization or the complexity of your service or system. </p>
<p> The principles discussed in this chapter can be tied together into a philosophy on
monitoring and alerting that’s widely endorsed and followed within Google SRE teams.
While this monitoring philosophy is a bit aspirational, it’s a good starting point
for writing or reviewing a new alert, and it can help your organization ask the
right questions, regardless of the size of your organization or the complexity of
your service or system. </p>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="creating rules for" id="id-wqC7SDIvfj"></a>When creating rules for monitoring and alerting, asking the following questions can help you avoid false positives and pager burnout:<sup><a data-type="noteref" id="id-a82udF8IBfx-marker" href="#id-a82udF8IBfx">24</a></sup>
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="creating rules for" id="id-wqC7SDIvfj"></a>When creating rules
for monitoring and alerting, asking the following questions can help you avoid false
positives and pager burnout:<sup><a data-type="noteref" id="id-a82udF8IBfx-marker"
href="#id-a82udF8IBfx">24</a></sup>
</p>
<ul>
<li>Does this rule detect <em>an otherwise undetected condition</em> that is urgent, actionable, and actively or imminently user-visible?<sup><a data-type="noteref" id="id-0vYuEFpSjSMtLfG-marker" href="#id-0vYuEFpSjSMtLfG">25</a></sup>
<li>Does this rule detect <em>an otherwise undetected condition</em> that is urgent,
actionable, and actively or imminently user-visible?<sup><a data-type="noteref"
id="id-0vYuEFpSjSMtLfG-marker"
href="#id-0vYuEFpSjSMtLfG">25</a></sup>
</li>
<li>Will I ever be able to ignore this alert, knowing it’s benign? When and why will
I be able to ignore this alert, and how can I avoid this scenario?
</li>
<li>Does this alert definitely indicate that users are being negatively affected?
Are there detectable cases in which users aren’t being negatively impacted, such
as drained traffic or test deployments, that should be filtered out?
</li>
<li>Can I take action in response to this alert? Is that action urgent, or could it
wait until morning? Could the action be safely automated? Will that action be a
long-term fix, or just a short-term workaround?
</li>
<li>Are other people getting paged for this issue, therefore rendering at least one
of the pages unnecessary?
</li>
<li>Will I ever be able to ignore this alert, knowing it’s benign? When and why will I be able to ignore this alert, and how can I avoid this scenario? </li>
<li>Does this alert definitely indicate that users are being negatively affected? Are there detectable cases in which users aren’t being negatively impacted, such as drained traffic or test deployments, that should be filtered out? </li>
<li>Can I take action in response to this alert? Is that action urgent, or could it wait until morning? Could the action be safely automated? Will that action be a long-term fix, or just a short-term workaround? </li>
<li>Are other people getting paged for this issue, therefore rendering at least one of the pages unnecessary? </li>
</ul>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="monitoring philosophy" id="id-PnCpSwhJfa"></a>These questions reflect a fundamental philosophy on pages and pagers:
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="monitoring philosophy" id="id-PnCpSwhJfa"></a>These questions
reflect a fundamental philosophy on pages and pagers:
</p>
<ul>
<li>Every time the pager goes off, I should be able to react with a sense of urgency. I can only react with a sense of urgency a few times a day before I become fatigued. </li>
<li>Every page should be actionable. </li>
<li>Every page response should require intelligence. If a page merely merits a robotic response, it shouldn’t be a page. </li>
<li>Pages should be about a novel problem or an event that hasn’t been seen before. </li>
<li>Every time the pager goes off, I should be able to react with a sense of
urgency. I can only react with a sense of urgency a few times a day before I
become fatigued.
</li>
<li>Every page should be actionable.</li>
<li>Every page response should require intelligence. If a page merely merits a
robotic response, it shouldn’t be a page.
</li>
<li>Pages should be about a novel problem or an event that hasn’t been seen
before.
</li>
</ul>
<p> Such a perspective dissipates certain distinctions: if a page satisfies the preceding four bullets, it’s irrelevant whether the page is triggered by white-box or black-box monitoring. This perspective also amplifies certain distinctions: it’s better to spend much more effort on catching symptoms than causes; when it comes to causes, only worry about very definite, very imminent causes. </p>
<p> Such a perspective dissipates certain distinctions: if a page satisfies the
preceding four bullets, it’s irrelevant whether the page is triggered by white-box
or black-box monitoring. This perspective also amplifies certain distinctions: it’s
better to spend much more effort on catching symptoms than causes; when it comes to
causes, only worry about very definite, very imminent causes. </p>
</section>
<section data-type="sect1" id="monitoring-for-the-long-term-NbsNS8">
<h2> Monitoring for the Long Term </h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="challenges of" id="id-wqC7SPFMSj"></a>In modern production systems, monitoring systems track an ever-evolving system with changing software architecture, load characteristics, and performance targets. An alert that’s currently exceptionally rare and hard to automate might become frequent, perhaps even meriting a hacked-together script to resolve it. At this point, someone should find and eliminate the root causes of the problem; if such resolution isn’t possible, the alert response deserves to be fully automated.
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="challenges of" id="id-wqC7SPFMSj"></a>In modern production
systems, monitoring systems track an ever-evolving system with changing software
architecture, load characteristics, and performance targets. An alert that’s
currently exceptionally rare and hard to automate might become frequent, perhaps
even meriting a hacked-together script to resolve it. At this point, someone should
find and eliminate the root causes of the problem; if such resolution isn’t
possible, the alert response deserves to be fully automated.
</p>
<p> It’s important that decisions about monitoring be made with long-term goals in mind. Every page that happens today distracts a human from improving the system for tomorrow, so there is often a case for taking a short-term hit to availability or performance in order to improve the long-term outlook for the system. Let’s take a look at two case studies that illustrate this trade-off. </p>
<p> It’s important that decisions about monitoring be made with long-term goals in mind.
Every page that happens today distracts a human from improving the system for
tomorrow, so there is often a case for taking a short-term hit to availability or
performance in order to improve the long-term outlook for the system. Let’s take a
look at two case studies that illustrate this trade-off. </p>
<section data-type="sect2" id="bigtable-sre-a-tale-of-over-alerting-dbsXtjSM">
<p>
<a data-type="indexterm" id="MDSbig6" data-primary="monitoring distributed systems" data-secondary="case studies"></a><a data-type="indexterm" data-primary="Bigtable" id="id-XmCpFOFytySv"></a>Google’s internal infrastructure is typically offered and measured against a service level objective (SLO; see <a data-type="xref" href="http://fakehost/sre/sre-book/chapters/service-level-objectives">Service Level Objectives</a>). Many years ago, the Bigtable service’s SLO was based on a synthetic well-behaved client’s mean performance. Because of problems in Bigtable and lower layers of the storage stack, the mean performance was driven by a "large" tail: the worst 5% of requests were often significantly slower than the rest.
<a data-type="indexterm" id="MDSbig6"
data-primary="monitoring distributed systems"
data-secondary="case studies"></a><a data-type="indexterm"
data-primary="Bigtable"
id="id-XmCpFOFytySv"></a>Google’s
internal infrastructure is typically offered and measured against a service
level objective (SLO; see <a data-type="xref"
href="http://fakehost/sre/sre-book/chapters/service-level-objectives">Service
Level Objectives</a>). Many years ago, the Bigtable service’s SLO was based on a
synthetic well-behaved client’s mean performance. Because of problems in
Bigtable and lower layers of the storage stack, the mean performance was driven
by a "large" tail: the worst 5% of requests were often significantly slower than
the rest.
</p>
<p> Email alerts were triggered as the SLO approached, and paging alerts were triggered when the SLO was exceeded. Both types of alerts were firing voluminously, consuming unacceptable amounts of engineering time: the team spent significant amounts of time triaging the alerts to find the few that were really actionable, and we often missed the problems that actually affected users, because so few of them did. Many of the pages were non-urgent, due to well-understood problems in the infrastructure, and had either rote responses or received no response. </p>
<p> To remedy the situation, the team used a three-pronged approach: while making great efforts to improve the performance of Bigtable, we also temporarily dialed back our SLO target, using the 75th percentile request latency. We also disabled email alerts, as there were so many that spending time diagnosing them was infeasible. </p>
<p> This strategy gave us enough breathing room to actually fix the longer-term problems in Bigtable and the lower layers of the storage stack, rather than constantly fixing tactical problems. On-call engineers could actually accomplish work when they weren’t being kept up by pages at all hours. Ultimately, temporarily backing off on our alerts allowed us to make faster progress toward a better service. </p>
<p> Email alerts were triggered as the SLO approached, and paging alerts were
triggered when the SLO was exceeded. Both types of alerts were firing
voluminously, consuming unacceptable amounts of engineering time: the team spent
significant amounts of time triaging the alerts to find the few that were really
actionable, and we often missed the problems that actually affected users,
because so few of them did. Many of the pages were non-urgent, due to
well-understood problems in the infrastructure, and had either rote responses or
received no response. </p>
<p> To remedy the situation, the team used a three-pronged approach: while making
great efforts to improve the performance of Bigtable, we also temporarily dialed
back our SLO target, using the 75th percentile request latency. We also disabled
email alerts, as there were so many that spending time diagnosing them was
infeasible. </p>
<p> This strategy gave us enough breathing room to actually fix the longer-term
problems in Bigtable and the lower layers of the storage stack, rather than
constantly fixing tactical problems. On-call engineers could actually accomplish
work when they weren’t being kept up by pages at all hours. Ultimately,
temporarily backing off on our alerts allowed us to make faster progress toward
a better service. </p>
</section>
<section data-type="sect2" id="gmail-predictable-scriptable-responses-from-humans-BVs1h4SD">
<section data-type="sect2"
id="gmail-predictable-scriptable-responses-from-humans-BVs1h4SD">
<p>
<a data-type="indexterm" data-primary="Gmail" id="id-XmC9SOFZhySv"></a>In the very early days of Gmail, the service was built on a retrofitted distributed process management system called Workqueue, which was originally created for batch processing of pieces of the search index. Workqueue was "adapted" to long-lived processes and subsequently applied to Gmail, but certain bugs in the relatively opaque codebase in the scheduler proved hard to beat.
<a data-type="indexterm" data-primary="Gmail" id="id-XmC9SOFZhySv"></a>In the
very early days of Gmail, the service was built on a retrofitted distributed
process management system called Workqueue, which was originally created for
batch processing of pieces of the search index. Workqueue was "adapted" to
long-lived processes and subsequently applied to Gmail, but certain bugs in the
relatively opaque codebase in the scheduler proved hard to beat.
</p>
<p> At that time, the Gmail monitoring was structured such that alerts fired when individual tasks were “de-scheduled” by Workqueue. This setup was less than ideal because even at that time, Gmail had many, many thousands of tasks, each task representing a fraction of a percent of our users. We cared deeply about providing a good user experience for Gmail users, but such an alerting setup was unmaintainable. </p>
<p> To address this problem, Gmail SRE built a tool that helped “poke” the scheduler in just the right way to minimize impact to users. The team had several discussions about whether or not we should simply automate the entire loop from detecting the problem to nudging the rescheduler, until a better long-term solution was achieved, but some worried this kind of workaround would delay a real fix. </p>
<p> This kind of tension is common within a team, and often reflects an underlying mistrust of the team’s self-discipline: while some team members want to implement a “hack” to allow time for a proper fix, others worry that a hack will be forgotten or that the proper fix will be deprioritized indefinitely. This concern is credible, as it’s easy to build layers of unmaintainable technical debt by patching over problems instead of making real fixes. Managers and technical leaders play a key role in implementing true, long-term fixes by supporting and prioritizing potentially time-consuming long-term fixes even when the initial “pain” of paging subsides. </p>
<p> Pages with rote, algorithmic responses should be a red flag. Unwillingness on the part of your team to automate such pages implies that the team lacks confidence that they can clean up their technical debt. This is a major problem worth escalating.<a data-type="indexterm" data-primary="" data-startref="MDSbig6" id="id-oPCASqT2hLSk"></a>
<p> At that time, the Gmail monitoring was structured such that alerts fired when
individual tasks were “de-scheduled” by Workqueue. This setup was less than
ideal because even at that time, Gmail had many, many thousands of tasks, each
task representing a fraction of a percent of our users. We cared deeply about
providing a good user experience for Gmail users, but such an alerting setup was
unmaintainable. </p>
<p> To address this problem, Gmail SRE built a tool that helped “poke” the scheduler
in just the right way to minimize impact to users. The team had several
discussions about whether or not we should simply automate the entire loop from
detecting the problem to nudging the rescheduler, until a better long-term
solution was achieved, but some worried this kind of workaround would delay a
real fix. </p>
<p> This kind of tension is common within a team, and often reflects an underlying
mistrust of the team’s self-discipline: while some team members want to
implement a “hack” to allow time for a proper fix, others worry that a hack will
be forgotten or that the proper fix will be deprioritized indefinitely. This
concern is credible, as it’s easy to build layers of unmaintainable technical
debt by patching over problems instead of making real fixes. Managers and
technical leaders play a key role in implementing true, long-term fixes by
supporting and prioritizing potentially time-consuming long-term fixes even when
the initial “pain” of paging subsides. </p>
<p> Pages with rote, algorithmic responses should be a red flag. Unwillingness on
the part of your team to automate such pages implies that the team lacks
confidence that they can clean up their technical debt. This is a major problem
worth escalating.<a data-type="indexterm" data-primary=""
data-startref="MDSbig6" id="id-oPCASqT2hLSk"></a>
</p>
</section>
<section data-type="sect2" id="the-long-run-MQsWTMS7">
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="short- vs. long-term availability" id="id-jyCxSoFETNSd"></a>A common theme connects the previous examples of Bigtable and Gmail: a tension between short-term and long-term availability. Often, sheer force of effort can help a rickety system achieve high availability, but this path is usually short-lived and fraught with burnout and dependence on a small number of heroic team members. Taking a controlled, short-term decrease in availability is often a painful, but strategic trade for the long-run stability of the system. It’s important not to think of every page as an event in isolation, but to consider whether the overall <em>level</em> of paging leads toward a healthy, appropriately available system with a healthy, viable team and long-term outlook. We review statistics about page frequency (usually expressed as incidents per shift, where an incident might be composed of a few related pages) in quarterly reports with management, ensuring that decision makers are kept up to date on the pager load and overall health of their teams.
<a data-type="indexterm" data-primary="monitoring distributed systems"
data-secondary="short- vs. long-term availability" id="id-jyCxSoFETNSd"></a>A
common theme connects the previous examples of Bigtable and Gmail: a tension
between short-term and long-term availability. Often, sheer force of effort can
help a rickety system achieve high availability, but this path is usually
short-lived and fraught with burnout and dependence on a small number of heroic
team members. Taking a controlled, short-term decrease in availability is often
a painful, but strategic trade for the long-run stability of the system. It’s
important not to think of every page as an event in isolation, but to consider
whether the overall <em>level</em> of paging leads toward a healthy,
appropriately available system with a healthy, viable team and long-term
outlook. We review statistics about page frequency (usually expressed as
incidents per shift, where an incident might be composed of a few related pages)
in quarterly reports with management, ensuring that decision makers are kept up
to date on the pager load and overall health of their teams.
</p>
</section>
</section>
<section data-type="sect1" id="conclusion-8ksvFj">
<h2> Conclusion </h2>
<p> A healthy monitoring and alerting pipeline is simple and easy to reason about. It focuses primarily on symptoms for paging, reserving cause-oriented heuristics to serve as aids to debugging problems. Monitoring symptoms is easier the further "up" your stack you monitor, though monitoring saturation and performance of subsystems such as databases often must be performed directly on the subsystem itself. Email alerts are of very limited value and tend to easily become overrun with noise; instead, you should favor a dashboard that monitors all ongoing subcritical problems for the sort of information that typically ends up in email alerts. A dashboard might also be paired with a log, in order to analyze historical correlations. </p>
<p> Over the long haul, achieving a successful on-call rotation and product includes choosing to alert on symptoms or imminent real problems, adapting your targets to goals that are actually achievable, and making sure that your monitoring supports rapid diagnosis. </p>
<p> A healthy monitoring and alerting pipeline is simple and easy to reason about. It
focuses primarily on symptoms for paging, reserving cause-oriented heuristics to
serve as aids to debugging problems. Monitoring symptoms is easier the further "up"
your stack you monitor, though monitoring saturation and performance of subsystems
such as databases often must be performed directly on the subsystem itself. Email
alerts are of very limited value and tend to easily become overrun with noise;
instead, you should favor a dashboard that monitors all ongoing subcritical problems
for the sort of information that typically ends up in email alerts. A dashboard
might also be paired with a log, in order to analyze historical correlations. </p>
<p> Over the long haul, achieving a successful on-call rotation and product includes
choosing to alert on symptoms or imminent real problems, adapting your targets to
goals that are actually achievable, and making sure that your monitoring supports
rapid diagnosis. </p>
</section>
</section>
</div>

1925
app/src/main/assets/readability/test/test-pages/google-sre-book-1/source.html
View File

File diff suppressed because it is too large Load Diff

795
app/src/main/assets/readability/test/test-pages/guardian-1/expected.html
View File

@ -1,31 +1,64 @@
<div id="readability-page-1" class="page">
<div itemprop="articleBody" data-test-id="article-review-body">
<p>
<span><span>W</span></span>hale whisperer Hori Parata was just seven years old when he attended his first mass stranding, a beaching of porpoises in New Zealand’s Northland, their cries screeching through the air on the deserted stretch of sand.
<span><span>W</span></span>hale whisperer Hori Parata was just seven years old when he
attended his first mass stranding, a beaching of porpoises in New Zealand’s Northland,
their cries screeching through the air on the deserted stretch of sand.
</p>
<p> Seven decades later, Parata, 75, has now overseen more than 500 strandings and is renowned in <a href="https://www.theguardian.com/world/newzealand" data-link-name="auto-linked-tag" data-component="auto-linked-tag">New Zealand</a> as the leading Māori whale expert, called on by tribes around the country for cultural guidance as marine strandings become increasingly complex and fatal. </p>
<p> “Man’s greed in the ocean is hurting the whales,” says Parata, a fierce and uncompromising elder of the Ngātiwai tribe of eastern Northland. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="05cb692c634cd90e5411aab92ca3e649474ff786" id="img-2">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f5b3d20f0dc5c22a83f96fb709ecd204" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3200" /><a href="#img-2" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> Seven decades later, Parata, 75, has now overseen more than 500 strandings and is
renowned in <a href="https://www.theguardian.com/world/newzealand"
data-link-name="auto-linked-tag" data-component="auto-linked-tag">New
Zealand</a> as the leading Māori whale expert, called on by tribes around the
country for cultural guidance as marine strandings become increasingly complex and
fatal. </p>
<p> “Man’s greed in the ocean is hurting the whales,” says Parata, a fierce and
uncompromising elder of the Ngātiwai tribe of eastern Northland. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="05cb692c634cd90e5411aab92ca3e649474ff786" id="img-2">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f5b3d20f0dc5c22a83f96fb709ecd204"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3200"/>
<a href="#img-2" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=adbbf5d870d9cc7f0b9a24eb5472ebf3 1760w" />
<source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f0a6d8fa60b5571e9e0e9a5673e407b7 880w" />
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d39897e20bb677fda3feb14113aad381 1600w" />
<source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=dbf240ee147c5a0a43321f1634ee41eb 800w" />
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f7926915234cc22c9fe718771f5837cc 1280w" />
<source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=fa3edf920739ca39d41e3fce38ab277b 640w" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f9fd7969943957bd4893fe29d248626c 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=61ab70443d54f672febc609b4bfbc5c0 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=a416bff5ba9e0d62f1634aee83308528 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5ae3b22ecb3c7bde8b49a212d52b707c 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c43d04fbea54b99991b541ce674da43d 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a0c813b07b8d5b99a33a7133ea7185db 445w" />
<img itemprop="contentUrl" alt="Hori Parata at his Pātaua farm, the place where he was born and grew up." src="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=575838a657b26493e956c7f84b058080" />
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)"
sizes="880px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=adbbf5d870d9cc7f0b9a24eb5472ebf3 1760w"/>
<source media="(min-width: 1300px)" sizes="880px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f0a6d8fa60b5571e9e0e9a5673e407b7 880w"/>
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)"
sizes="800px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d39897e20bb677fda3feb14113aad381 1600w"/>
<source media="(min-width: 1140px)" sizes="800px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=dbf240ee147c5a0a43321f1634ee41eb 800w"/>
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)"
sizes="640px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f7926915234cc22c9fe718771f5837cc 1280w"/>
<source media="(min-width: 980px)" sizes="640px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=fa3edf920739ca39d41e3fce38ab277b 640w"/>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f9fd7969943957bd4893fe29d248626c 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=61ab70443d54f672febc609b4bfbc5c0 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=a416bff5ba9e0d62f1634aee83308528 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5ae3b22ecb3c7bde8b49a212d52b707c 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c43d04fbea54b99991b541ce674da43d 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a0c813b07b8d5b99a33a7133ea7185db 445w"/>
<img itemprop="contentUrl"
alt="Hori Parata at his Pātaua farm, the place where he was born and grew up."
src="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=575838a657b26493e956c7f84b058080"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
@ -35,327 +68,633 @@
<p> Hori Parata at his Pātaua farm, the place where he was born and grew up </p>
</li>
</ul>
<p> “We’re having to put up with a lot of stuff today. The public want to hug the whales, they want to touch them, they want to feel good – that’s not the thing. We feel that is ridiculous.” </p>
<p> Whale experts regard New Zealand – or Aotearoa as it is called by Māori – as the whale stranding capital of the world, with more than 5,000 incidents recorded since 1840, and an average of 300 individual animals beaching themselves each year. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="98c683a7df9c83b2c13de2d93ca1825199ed5150" id="img-3">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2432083289db6bd42f6ad7adc64a78d" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3166" /><a href="#img-3" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> “We’re having to put up with a lot of stuff today. The public want to hug the whales,
they want to touch them, they want to feel good – that’s not the thing. We feel that is
ridiculous.” </p>
<p> Whale experts regard New Zealand – or Aotearoa as it is called by Māori – as the whale
stranding capital of the world, with more than 5,000 incidents recorded since 1840, and
an average of 300 individual animals beaching themselves each year. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="98c683a7df9c83b2c13de2d93ca1825199ed5150" id="img-3">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2432083289db6bd42f6ad7adc64a78d"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3166"/>
<a href="#img-3" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7234718ccc92b4251bdfe4c505f064f8 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b80f7e8fcfbbe8e12fa6954ddeb3bd1e 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=78ebc960d0c15f627bfbe60cd770a139 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1319684ba57c18162a45e56384d418b0 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3170b3a80d76d6c6422b65045444829e 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e835570164cc241d4009af47fc1051f7 445w" />
<img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, watched by his father Hori Parata, carves a traditional Maōri design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata" src="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2f198e1958f140f3ac664a3fdd87177c" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7234718ccc92b4251bdfe4c505f064f8 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b80f7e8fcfbbe8e12fa6954ddeb3bd1e 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=78ebc960d0c15f627bfbe60cd770a139 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1319684ba57c18162a45e56384d418b0 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3170b3a80d76d6c6422b65045444829e 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e835570164cc241d4009af47fc1051f7 445w"/>
<img itemprop="contentUrl"
alt="Kauri (Tekaurinui Robert) Parata, watched by his father Hori Parata, carves a traditional Maōri design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata"
src="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2f198e1958f140f3ac664a3fdd87177c"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<ul>
<li>
<p> Kauri (Te Kaurinui Robert) Parata, watched by his father, Hori Parata, carves a traditional Māori design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father </p>
<p> Kauri (Te Kaurinui Robert) Parata, watched by his father, Hori Parata, carves a
traditional Māori design at their home in Whangārei. Kauri is a member of the
Manu Taupunga group that is the organising arm of the whale-body recovery
operation started by his father </p>
</li>
</ul>
<p> Concrete information on why whales strand remains elusive, but “sickness, navigational error, geographical features, a rapidly falling tide, being chased by a predator, or extreme weather” are all thought to contribute, according to the New Zealand Department of Conservation. </p>
<p> Climate change is to blame too, <a href="https://www.radionz.co.nz/news/national/377272/new-zealand-beached-whales-why-are-so-many-getting-stranded" data-link-name="in body link">scientists think</a>, with warming ocean temperatures moving whales’ prey closer to the shore and forcing them to pursue their food into shallow waters. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="0447972cf47ca67882fcfc648edf7e574b0853bc" id="img-4">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0e7302276c6cc7cb65be9bcdacd3081f" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3200" /><a href="#img-4" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> Concrete information on why whales strand remains elusive, but “sickness, navigational
error, geographical features, a rapidly falling tide, being chased by a predator, or
extreme weather” are all thought to contribute, according to the New Zealand Department
of Conservation. </p>
<p> Climate change is to blame too, <a
href="https://www.radionz.co.nz/news/national/377272/new-zealand-beached-whales-why-are-so-many-getting-stranded"
data-link-name="in body link">scientists think</a>, with warming ocean temperatures
moving whales’ prey closer to the shore and forcing them to pursue their food into
shallow waters. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="0447972cf47ca67882fcfc648edf7e574b0853bc" id="img-4">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0e7302276c6cc7cb65be9bcdacd3081f"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3200"/>
<a href="#img-4" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=933c5693fbf9195f6e83a8928283f85e 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e07b988894308f3bfc052d1ff9dfc1e2 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=113cf92fe80bc5996634c34b3c7a0c09 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5a7358223e80941bd1b0e0f427beefde 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=292e941797de1671c185c1b074c688ad 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d1bc1c4d9341f9b6d63b64861a3de711 445w" />
<img itemprop="contentUrl" alt="A bin of small whale bones." src="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=718132fd888108a18383c24a8425523b" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=933c5693fbf9195f6e83a8928283f85e 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e07b988894308f3bfc052d1ff9dfc1e2 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=113cf92fe80bc5996634c34b3c7a0c09 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5a7358223e80941bd1b0e0f427beefde 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=292e941797de1671c185c1b074c688ad 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d1bc1c4d9341f9b6d63b64861a3de711 445w"/>
<img itemprop="contentUrl" alt="A bin of small whale bones."
src="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=718132fd888108a18383c24a8425523b"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="416800b8d06039780c3e6de28564e6f277b4e7b7" id="img-5">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=249ddc3119aa637c3a1ae4998509f604" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3200" /><a href="#img-5" data-link-name="Launch Article Lightbox" data-is-ajax="">
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="416800b8d06039780c3e6de28564e6f277b4e7b7" id="img-5">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=249ddc3119aa637c3a1ae4998509f604"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3200"/>
<a href="#img-5" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4197b7a2bc3c7e32d3eaee927b9d08e6 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=82d6908c6ffe622566e1cc6513d60ddd 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=222e300fb2a60da1a841fbbc2bc8f752 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=63ba1cfe4b5e7c7d741d311d58997fcf 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=231139825b311ede8d01e6c702d6d12c 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1065dd4439e14c72fe520c19566172e2 445w" />
<img itemprop="contentUrl" alt="The baleen recovered from a stranded Pygmy Right Whale." src="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c5a53178ebbe54490c97fad6b5e032c4" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4197b7a2bc3c7e32d3eaee927b9d08e6 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=82d6908c6ffe622566e1cc6513d60ddd 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=222e300fb2a60da1a841fbbc2bc8f752 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=63ba1cfe4b5e7c7d741d311d58997fcf 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=231139825b311ede8d01e6c702d6d12c 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1065dd4439e14c72fe520c19566172e2 445w"/>
<img itemprop="contentUrl"
alt="The baleen recovered from a stranded Pygmy Right Whale."
src="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c5a53178ebbe54490c97fad6b5e032c4"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="8c207197c0a9e6f407dcddfded5f868a142c9cab" id="img-6">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=bcfefc5593dcc2bdb0dd3b0543cfa4eb" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3200" /><a href="#img-6" data-link-name="Launch Article Lightbox" data-is-ajax="">
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="8c207197c0a9e6f407dcddfded5f868a142c9cab" id="img-6">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=bcfefc5593dcc2bdb0dd3b0543cfa4eb"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3200"/>
<a href="#img-6" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d272c4dc57408daa0dcdf5461a230e9e 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=890f0e0cee54663a90292471a16f95d2 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=59a57c5bded93fe0efe0046226fe7c69 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f1b2a4c79e965aa76a322ad25072a052 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=ffbdfd671eb45e3a87e3dc9137e8b006 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c560f0b0ef9736c1a66215cc29c59d43 445w" />
<img itemprop="contentUrl" alt="Squid beaks, from the stomach of a Sperm Whale." src="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0dd659ae339b1aea9a99ed7f6f8eadeb" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d272c4dc57408daa0dcdf5461a230e9e 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=890f0e0cee54663a90292471a16f95d2 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=59a57c5bded93fe0efe0046226fe7c69 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f1b2a4c79e965aa76a322ad25072a052 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=ffbdfd671eb45e3a87e3dc9137e8b006 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c560f0b0ef9736c1a66215cc29c59d43 445w"/>
<img itemprop="contentUrl"
alt="Squid beaks, from the stomach of a Sperm Whale."
src="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0dd659ae339b1aea9a99ed7f6f8eadeb"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<ul>
<li>
<p> Clockwise from top: small whale bones; squid beaks, from the stomach of a sperm whale; the baleen filter-feeder system recovered from a stranded pygmy right whale. </p>
<p> Clockwise from top: small whale bones; squid beaks, from the stomach of a sperm
whale; the baleen filter-feeder system recovered from a stranded pygmy right
whale. </p>
</li>
</ul>
<h2> ‘Unprecedented’ strandings </h2>
<p> November marked the beginning of whale stranding season, and it started with a surge in incidents, according to whale rescue group Project Jonah, with <a href="https://www.theguardian.com/environment/video/2018/nov/26/whales-die-stranded-stewart-island-new-zealand-beach-video" data-link-name="in body link">140 pilot whales</a> beaching and dying on Stewart Island, 10 rare pygmy whales on Ninety Mile beach, 51 stranded and dead on the Chatham Islands and a spate of individual cases around the country. </p>
<p> And as more whales beach and die – from exhaustion, heat stroke or seagulls feasting on their flesh – an acute sense of grief is growing among New Zealand’s indigenous people, who regard whales as their ancestors and <em>taonga</em> (treasures). </p>
<p> “These days it is like a zoo. People just want to come and gawk at us, without even trying to understand what is happening with the animals and the environment,” says Parata, bristling with anger. </p>
<figure data-interactive="https://interactive.guim.co.uk/embed/iframe-wrapper/0.1/boot.js" data-canonical-url="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/" data-alt="whale strandings">
<a href="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/" data-link-name="in body link">whale strandings</a>
<p> November marked the beginning of whale stranding season, and it started with a surge in
incidents, according to whale rescue group Project Jonah, with <a
href="https://www.theguardian.com/environment/video/2018/nov/26/whales-die-stranded-stewart-island-new-zealand-beach-video"
data-link-name="in body link">140 pilot whales</a> beaching and dying on Stewart
Island, 10 rare pygmy whales on Ninety Mile beach, 51 stranded and dead on the Chatham
Islands and a spate of individual cases around the country. </p>
<p> And as more whales beach and die – from exhaustion, heat stroke or seagulls feasting on
their flesh – an acute sense of grief is growing among New Zealand’s indigenous people,
who regard whales as their ancestors and <em>taonga</em> (treasures). </p>
<p> “These days it is like a zoo. People just want to come and gawk at us, without even
trying to understand what is happening with the animals and the environment,” says
Parata, bristling with anger. </p>
<figure data-interactive="https://interactive.guim.co.uk/embed/iframe-wrapper/0.1/boot.js"
data-canonical-url="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/"
data-alt="whale strandings">
<a href="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/"
data-link-name="in body link">whale strandings</a>
</figure>
<p> “When will we talk about what is hurting these animals out on the sea? They are drowning out there, they can’t breathe, they beach themselves to be with the Aunties.” </p>
<p> Ngātiwai believe the whales beach when they are ready to die and want to return to their families, the Māori people. Then, their human families use the whales’ gift of their bodies for sacred carvings, for traditional medicines, and even for compost. </p>
<p> There are marked tribal differences across New Zealand and while some tribes work to refloat stranded whales, others like Parata’s Ngātiwai stand back and allow the Department of Conservation and volunteer groups to take the lead in rescue efforts. </p>
<p> Then the tribe moves in en masse and holds a <em>karakia</em> (prayer), names each animal and sets to work removing their bones, blubber, eyes and teeth for cultural purposes. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="0d13adeb0790af5c5fa317ce477c323d0e1c773c" id="img-7">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d73916fda0f6c60b5eebcb71fea643e9" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="2334" /><a href="#img-7" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> “When will we talk about what is hurting these animals out on the sea? They are drowning
out there, they can’t breathe, they beach themselves to be with the Aunties.” </p>
<p> Ngātiwai believe the whales beach when they are ready to die and want to return to their
families, the Māori people. Then, their human families use the whales’ gift of their
bodies for sacred carvings, for traditional medicines, and even for compost. </p>
<p> There are marked tribal differences across New Zealand and while some tribes work to
refloat stranded whales, others like Parata’s Ngātiwai stand back and allow the
Department of Conservation and volunteer groups to take the lead in rescue efforts. </p>
<p> Then the tribe moves in en masse and holds a <em>karakia</em> (prayer), names each
animal and sets to work removing their bones, blubber, eyes and teeth for cultural
purposes. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="0d13adeb0790af5c5fa317ce477c323d0e1c773c" id="img-7">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d73916fda0f6c60b5eebcb71fea643e9"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="2334"/>
<a href="#img-7" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=524b92a6590a1b5c139bfe50581476a1 2600w" />
<source media="(min-width: 1300px)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=837d7df48e565ffad49febab20fbd179 1300w" />
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=fe98f4515ae3490f53804235a4dcf84b 2280w" />
<source media="(min-width: 1140px)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a3ec73ab7f20798ec264f5ecb59f6bcc 1140w" />
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3ac3fd319accb248cf8a13e8ef55ebcc 2250w" />
<source media="(min-width: 980px)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2561c80af0d0e8eed2fc7c535be48696 1125w" />
<source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)" sizes="965px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=539617a30ea852b01218eec69d6066e6 1930w" />
<source media="(min-width: 740px)" sizes="965px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=624139eae65dd4c96efd1a90243b9286 965w" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="725px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=633619afab6a0ee26a3839b69160e854 1450w" />
<source media="(min-width: 660px)" sizes="725px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98b9351340e8ee30063a4b6eb6501d6f 725w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="645px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e11a31f8adaab06b457fd9c7dda6bf6d 1290w" />
<source media="(min-width: 480px)" sizes="645px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=912104c53b38d1cd424bfb82302480e1 645w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="465px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=209259fab8ec9e81386838f2d5cdecc3 930w" />
<source media="(min-width: 0px)" sizes="465px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=06848a63793084a21bc86aa613b126a7 465w" />
<img itemprop="contentUrl" alt="Buck Cullen with his daughter Kaiarahi (10 months) in his back yard where he is storing a pair of massive Sperm Whale jawbones. Buck is a integral member of the whale recovery team, alongside Hori Parata." src="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=41b88ee343b9be76688b88443c7a8958" />
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)"
sizes="1300px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=524b92a6590a1b5c139bfe50581476a1 2600w"/>
<source media="(min-width: 1300px)" sizes="1300px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=837d7df48e565ffad49febab20fbd179 1300w"/>
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)"
sizes="1140px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=fe98f4515ae3490f53804235a4dcf84b 2280w"/>
<source media="(min-width: 1140px)" sizes="1140px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a3ec73ab7f20798ec264f5ecb59f6bcc 1140w"/>
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)"
sizes="1125px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3ac3fd319accb248cf8a13e8ef55ebcc 2250w"/>
<source media="(min-width: 980px)" sizes="1125px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2561c80af0d0e8eed2fc7c535be48696 1125w"/>
<source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)"
sizes="965px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=539617a30ea852b01218eec69d6066e6 1930w"/>
<source media="(min-width: 740px)" sizes="965px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=624139eae65dd4c96efd1a90243b9286 965w"/>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="725px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=633619afab6a0ee26a3839b69160e854 1450w"/>
<source media="(min-width: 660px)" sizes="725px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98b9351340e8ee30063a4b6eb6501d6f 725w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="645px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e11a31f8adaab06b457fd9c7dda6bf6d 1290w"/>
<source media="(min-width: 480px)" sizes="645px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=912104c53b38d1cd424bfb82302480e1 645w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="465px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=209259fab8ec9e81386838f2d5cdecc3 930w"/>
<source media="(min-width: 0px)" sizes="465px"
srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=06848a63793084a21bc86aa613b126a7 465w"/>
<img itemprop="contentUrl"
alt="Buck Cullen with his daughter Kaiarahi (10 months) in his back yard where he is storing a pair of massive Sperm Whale jawbones. Buck is a integral member of the whale recovery team, alongside Hori Parata."
src="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=41b88ee343b9be76688b88443c7a8958"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<ul>
<li>
<p> Buck Cullen with his daughter Kaiarahi (10 months) in his backyard, where he is storing a pair of massive sperm whale jawbones. Cullen is an integral member of Hori Parata’s whale recovery team </p>
<p> Buck Cullen with his daughter Kaiarahi (10 months) in his backyard, where he is
storing a pair of massive sperm whale jawbones. Cullen is an integral member of
Hori Parata’s whale recovery team </p>
</li>
</ul>
<p> But indigenous elders say they aren’t being listened to when they tell the government their whale kin are sick, and trying to escape an increasingly polluted and unpredictable ocean. </p>
<p> Earlier this year in South Taranaki, a mass stranding that was described as <a href="https://www.stuff.co.nz/national/104249829/unprecedented-whale-strandings-reaches-11-in-total-on-taranaki-beach" data-link-name="in body link">“unprecedented”</a> left the local Māori tribe scrambling. Security was brought in when thieves attacked a sperm whale with an axe, trying to remove valuable teeth from its jaw. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="4973b41f53b8ade499f99a305b01157eca659ca5" id="img-8">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ee921715aaf10e0816dcf15dfc3a21f5" />
<meta itemprop="width" content="1200" />
<meta itemprop="height" content="900" /><a href="#img-8" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> But indigenous elders say they aren’t being listened to when they tell the government
their whale kin are sick, and trying to escape an increasingly polluted and
unpredictable ocean. </p>
<p> Earlier this year in South Taranaki, a mass stranding that was described as <a
href="https://www.stuff.co.nz/national/104249829/unprecedented-whale-strandings-reaches-11-in-total-on-taranaki-beach"
data-link-name="in body link">“unprecedented”</a> left the local Māori tribe
scrambling. Security was brought in when thieves attacked a sperm whale with an axe,
trying to remove valuable teeth from its jaw. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="4973b41f53b8ade499f99a305b01157eca659ca5" id="img-8">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ee921715aaf10e0816dcf15dfc3a21f5"/>
<meta itemprop="width" content="1200"/>
<meta itemprop="height" content="900"/>
<a href="#img-8" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8eb5af62092c8a0eb0d283cb0887f5c5 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9ace0f595be568263c5991993632dae8 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=97219f131f1013d15fed7c63308f2b2f 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=17191691b6cd81c2a67730d5475db08b 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=30d51b75767551399ba174bae5c39e94 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=936568b6e7da0a710abcd2c015fd0a8b 445w" />
<img itemprop="contentUrl" alt="12 Parāoa Whales (Sperm Whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen on their coast in recent memory." src="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7c255bf6f8a27c56365a86813cdd1517" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8eb5af62092c8a0eb0d283cb0887f5c5 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9ace0f595be568263c5991993632dae8 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=97219f131f1013d15fed7c63308f2b2f 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=17191691b6cd81c2a67730d5475db08b 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=30d51b75767551399ba174bae5c39e94 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=936568b6e7da0a710abcd2c015fd0a8b 445w"/>
<img itemprop="contentUrl"
alt="12 Parāoa Whales (Sperm Whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen on their coast in recent memory."
src="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7c255bf6f8a27c56365a86813cdd1517"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<ul>
<li>
<p> 12 parāoa whales (sperm whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen near this location in recent memory </p>
<p> 12 parāoa whales (sperm whales) recently stranded on the South Taranaki coast of
Kaupokonui, on a scale not seen near this location in recent memory </p>
</li>
</ul>
<p> Parata and his 22-year-old son, Te Kaurinui Robert Parata, were called in to assist. Te Kaurinui was called after the first whale his father ever named, and left university this year to return to Whangārei and study whale <em>tikanga</em> (protocol) and carving. </p>
<p> He says mass strandings are getting more local and international attention and money from donations, but traditional knowledge is being dismissed as overly spiritual. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="d1941b6a6908314fab28f44da222a4c892213341" id="img-9">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4e46b70079819caee86359a058c92be1" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3120" /><a href="#img-9" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> Parata and his 22-year-old son, Te Kaurinui Robert Parata, were called in to assist. Te
Kaurinui was called after the first whale his father ever named, and left university
this year to return to Whangārei and study whale <em>tikanga</em> (protocol) and
carving. </p>
<p> He says mass strandings are getting more local and international attention and money
from donations, but traditional knowledge is being dismissed as overly spiritual. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="d1941b6a6908314fab28f44da222a4c892213341" id="img-9">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4e46b70079819caee86359a058c92be1"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3120"/>
<a href="#img-9" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b775d649981f7bc36cf0753dd9f47862 1760w" />
<source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3aa0de336dad5991ff92d488288fcc92 880w" />
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9133cf8cd42d42bb1342dbefbfc67c22 1600w" />
<source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2916efc01252a457768bfd28b40591ee 800w" />
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c7044b93d3aa73f7eac1b56ab9d77d86 1280w" />
<source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2342e6519bfe1ce74a04ca96c06ed4cd 640w" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b6218129f26e09d8133bf226df3e9bed 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9d20a1ca789f2efe7ba2dfdef9c28725 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=73fb4a4f8d59a92325fef9faa2ddf90d 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7894b6600fbb6af1ec6552007ad12da8 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=19022ce9fcafebe383880059f34add78 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ded05f2c1859d70d3b8d3de3dcb75e0f 445w" />
<img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, of the New Zealand Māori tribe Ngāti Wai, in front of the carving shed at Hihiaua Cultural Centre in Whangarei" src="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=76a26a289728e3d625e57d32eced57d8" />
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)"
sizes="880px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b775d649981f7bc36cf0753dd9f47862 1760w"/>
<source media="(min-width: 1300px)" sizes="880px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3aa0de336dad5991ff92d488288fcc92 880w"/>
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)"
sizes="800px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9133cf8cd42d42bb1342dbefbfc67c22 1600w"/>
<source media="(min-width: 1140px)" sizes="800px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2916efc01252a457768bfd28b40591ee 800w"/>
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)"
sizes="640px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c7044b93d3aa73f7eac1b56ab9d77d86 1280w"/>
<source media="(min-width: 980px)" sizes="640px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2342e6519bfe1ce74a04ca96c06ed4cd 640w"/>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b6218129f26e09d8133bf226df3e9bed 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9d20a1ca789f2efe7ba2dfdef9c28725 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=73fb4a4f8d59a92325fef9faa2ddf90d 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7894b6600fbb6af1ec6552007ad12da8 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=19022ce9fcafebe383880059f34add78 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ded05f2c1859d70d3b8d3de3dcb75e0f 445w"/>
<img itemprop="contentUrl"
alt="Kauri (Tekaurinui Robert) Parata, of the New Zealand Māori tribe Ngāti Wai, in front of the carving shed at Hihiaua Cultural Centre in Whangarei"
src="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=76a26a289728e3d625e57d32eced57d8"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<ul>
<li>
<p> Clockwise from top: Te Kaurinui Parata, in front of the carving shed at Hihiaua Cultural Centre in Whangārei; Parata holds three whale teeth recovered from a beached whale – the middle one shows marks where a poacher had attempted to hack it out with an axe before the recovery group arrived; the Pou, a tribal identifier, in front of the carving shed. </p>
<p> Clockwise from top: Te Kaurinui Parata, in front of the carving shed at Hihiaua
Cultural Centre in Whangārei; Parata holds three whale teeth recovered from a
beached whale – the middle one shows marks where a poacher had attempted to hack
it out with an axe before the recovery group arrived; the Pou, a tribal
identifier, in front of the carving shed. </p>
</li>
</ul>
<h2> ‘We need to listen’ </h2>
<p> Māori harvest rights over dead whales have only been officially recognised since 1998, and the practice still elicits horror from some New Zealanders and visitors. </p>
<p> “Our own ancestors wouldn’t say to go down there and hug the whales. That’s a modern thing,” says Te Kaurinui. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="14b462f5d9489def554e0f9f436f13aec332f7b8" id="img-10">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b8bf0c299908608310fb47e4163afb38" />
<meta itemprop="width" content="4278" />
<meta itemprop="height" content="4800" /><a href="#img-10" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> Māori harvest rights over dead whales have only been officially recognised since 1998,
and the practice still elicits horror from some New Zealanders and visitors. </p>
<p> “Our own ancestors wouldn’t say to go down there and hug the whales. That’s a modern
thing,” says Te Kaurinui. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="14b462f5d9489def554e0f9f436f13aec332f7b8" id="img-10">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b8bf0c299908608310fb47e4163afb38"/>
<meta itemprop="width" content="4278"/>
<meta itemprop="height" content="4800"/>
<a href="#img-10" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=1a392a17d0c74d513ff14129ae5f7e6c 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d5b05458659a4a06ff6583046f712ad1 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=649c2127cfbab69a7d6e4085ea098d95 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d45dfb664b5479650ba067c6a5af16c3 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f94f4c652479979124e52c237804eb83 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f6c8d3b38a8af6d197d0a60d22a326f3 445w" />
<img itemprop="contentUrl" alt="The Pou in front of the carving shed at Hihiaua Cultural centre" src="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a45ec7578a392eec201d2f6920b609a0" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=1a392a17d0c74d513ff14129ae5f7e6c 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d5b05458659a4a06ff6583046f712ad1 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=649c2127cfbab69a7d6e4085ea098d95 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d45dfb664b5479650ba067c6a5af16c3 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f94f4c652479979124e52c237804eb83 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f6c8d3b38a8af6d197d0a60d22a326f3 445w"/>
<img itemprop="contentUrl"
alt="The Pou in front of the carving shed at Hihiaua Cultural centre"
src="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a45ec7578a392eec201d2f6920b609a0"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="e2cf54c36f17c6894844ea0cdd4346288a002da9" id="img-11">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3691946929b4105ba35ff9d306cb4fbc" />
<meta itemprop="width" content="3172" />
<meta itemprop="height" content="3189" /><a href="#img-11" data-link-name="Launch Article Lightbox" data-is-ajax="">
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="e2cf54c36f17c6894844ea0cdd4346288a002da9" id="img-11">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3691946929b4105ba35ff9d306cb4fbc"/>
<meta itemprop="width" content="3172"/>
<meta itemprop="height" content="3189"/>
<a href="#img-11" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=be6cf9beea564e7be872193b01c329a3 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c74497ec55bb1b680169d62684aa803d 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=06b2c03638556f85da582e419cf1817a 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8cc04403e3902dbe1e4a9b01b8d4e517 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=893cbf39d6db1d28ba8fc85419382f9d 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6df725aaf59e21b09434b967e05b3272 445w" />
<img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, holds three whale teeth recovered from a beached whale. The middle tooth shows the marks where a poacher had attempted to hack it out with an axe before the recovery group arrived. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata." src="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0bd8e9f51bdf79a6e0a15ed176cfb57d" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=be6cf9beea564e7be872193b01c329a3 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c74497ec55bb1b680169d62684aa803d 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=06b2c03638556f85da582e419cf1817a 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8cc04403e3902dbe1e4a9b01b8d4e517 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=893cbf39d6db1d28ba8fc85419382f9d 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6df725aaf59e21b09434b967e05b3272 445w"/>
<img itemprop="contentUrl"
alt="Kauri (Tekaurinui Robert) Parata, holds three whale teeth recovered from a beached whale. The middle tooth shows the marks where a poacher had attempted to hack it out with an axe before the recovery group arrived. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata."
src="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0bd8e9f51bdf79a6e0a15ed176cfb57d"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<p> The Ngātiwai are investigating a possible link between the crisis of the dieback disease killing New Zealand’s native kauri trees – and <a href="https://www.theguardian.com/world/2018/jul/14/like-losing-family-time-may-be-running-out-for-new-zealands-most-sacred-tree" data-link-name="in body link">threatening the giant Tāne Mahuta, which may be 2,000 years old</a> – and the increase in whale strandings. </p>
<p> Parata and his family believe whale oil and byproducts could be used to try to cure Kauri dieback, and want more government money and attention directed towards indigenous knowledge of the interconnectedness of the New Zealand environment, and possible indigenous solutions. </p>
<p> “People dismiss us when we tell them our spiritual understanding of whales – why they are beaching, why they are hurting,” says Te Kaurinui. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="b5f3736b2ba2ef4df364258b0efcaba26f571d6e" id="img-12">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=72146577e379bdf9a21bc47994de5fb6" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3073" /><a href="#img-12" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> The Ngātiwai are investigating a possible link between the crisis of the dieback disease
killing New Zealand’s native kauri trees – and <a
href="https://www.theguardian.com/world/2018/jul/14/like-losing-family-time-may-be-running-out-for-new-zealands-most-sacred-tree"
data-link-name="in body link">threatening the giant Tāne Mahuta, which may be
2,000 years old</a> – and the increase in whale strandings. </p>
<p> Parata and his family believe whale oil and byproducts could be used to try to cure
Kauri dieback, and want more government money and attention directed towards indigenous
knowledge of the interconnectedness of the New Zealand environment, and possible
indigenous solutions. </p>
<p> “People dismiss us when we tell them our spiritual understanding of whales – why they
are beaching, why they are hurting,” says Te Kaurinui. </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="b5f3736b2ba2ef4df364258b0efcaba26f571d6e" id="img-12">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=72146577e379bdf9a21bc47994de5fb6"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3073"/>
<a href="#img-12" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4bd713ae6f60b56e36378bb2fe45e9eb 2600w" />
<source media="(min-width: 1300px)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b93c6d33ca7496220eb30d0c8bfaccf9 1300w" />
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f66f5a256170eced3c0a52aee235ae29 2280w" />
<source media="(min-width: 1140px)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1f60eb7cf5fc6f899c7c9af9c05df9a0 1140w" />
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=27526c9caf1a62110dfa16214ceccf14 2250w" />
<source media="(min-width: 980px)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7d47ba3d7898a398e1f9d8f2d19d391e 1125w" />
<source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)" sizes="965px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8974cfeb3e33bc51b55518f1bc81e68d 1930w" />
<source media="(min-width: 740px)" sizes="965px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=04f953b4ec9555178204a1621b8a1f59 965w" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="725px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7e2c1494173f83f555806120a1fe43cc 1450w" />
<source media="(min-width: 660px)" sizes="725px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8f53cf25b514e283cce3d995220ca19b 725w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="645px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2b770ca04273c921c8abe0a7de72030a 1290w" />
<source media="(min-width: 480px)" sizes="645px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0da775328d9ade5ca605079a9118a64a 645w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="465px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e2e51037c5a6b8bf22d87cb927e45888 930w" />
<source media="(min-width: 0px)" sizes="465px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7ecf73b514ceb1b5e0391f17c7a4d3b0 465w" />
<img itemprop="contentUrl" alt="Whangārei Harbour from Tamaterau, looking south through Mangrove sprouts coming up through the harbourside silt." src="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7fcadc35b3a44ebafe3c469c6e89241d" />
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)"
sizes="1300px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4bd713ae6f60b56e36378bb2fe45e9eb 2600w"/>
<source media="(min-width: 1300px)" sizes="1300px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b93c6d33ca7496220eb30d0c8bfaccf9 1300w"/>
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)"
sizes="1140px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f66f5a256170eced3c0a52aee235ae29 2280w"/>
<source media="(min-width: 1140px)" sizes="1140px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1f60eb7cf5fc6f899c7c9af9c05df9a0 1140w"/>
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)"
sizes="1125px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=27526c9caf1a62110dfa16214ceccf14 2250w"/>
<source media="(min-width: 980px)" sizes="1125px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7d47ba3d7898a398e1f9d8f2d19d391e 1125w"/>
<source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)"
sizes="965px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8974cfeb3e33bc51b55518f1bc81e68d 1930w"/>
<source media="(min-width: 740px)" sizes="965px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=04f953b4ec9555178204a1621b8a1f59 965w"/>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="725px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7e2c1494173f83f555806120a1fe43cc 1450w"/>
<source media="(min-width: 660px)" sizes="725px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8f53cf25b514e283cce3d995220ca19b 725w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="645px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2b770ca04273c921c8abe0a7de72030a 1290w"/>
<source media="(min-width: 480px)" sizes="645px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0da775328d9ade5ca605079a9118a64a 645w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="465px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e2e51037c5a6b8bf22d87cb927e45888 930w"/>
<source media="(min-width: 0px)" sizes="465px"
srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7ecf73b514ceb1b5e0391f17c7a4d3b0 465w"/>
<img itemprop="contentUrl"
alt="Whangārei Harbour from Tamaterau, looking south through Mangrove sprouts coming up through the harbourside silt."
src="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7fcadc35b3a44ebafe3c469c6e89241d"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<ul>
<li>
<p> Whangārei Harbour seen from Tamaterau, with mangrove sprouts coming up through the harbourside silt </p>
<p> Whangārei Harbour seen from Tamaterau, with mangrove sprouts coming up through
the harbourside silt </p>
</li>
</ul>
<p> “We are not foreigners in this land. We did not take this land off anyone else. We were not lost waiting for some bullheads to tell us what was going on.” </p>
<p> Kaitaia conservation department ranger Jamie Werner of Ngātiwai recently attended his first mass beaching on Ninety Mile Beach. It was the first recorded time pygmy whales had stranded on New Zealand shores. </p>
<p> “I arrived at the beach and we leapfrogged between the animals. They were calling out to each other and reassuring each other,” says Werner. “It was a shock. We’re working to adapt but the ocean is changing so fast.” </p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="d5aaf60e3a427f278747acf0c3e7ba39b39ef923" id="img-13">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1d00d3ea91d32110b40d85ff43227728" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3200" /><a href="#img-13" data-link-name="Launch Article Lightbox" data-is-ajax="">
<p> “We are not foreigners in this land. We did not take this land off anyone else. We were
not lost waiting for some bullheads to tell us what was going on.” </p>
<p> Kaitaia conservation department ranger Jamie Werner of Ngātiwai recently attended his
first mass beaching on Ninety Mile Beach. It was the first recorded time pygmy whales
had stranded on New Zealand shores. </p>
<p> “I arrived at the beach and we leapfrogged between the animals. They were calling out to
each other and reassuring each other,” says Werner. “It was a shock. We’re working to
adapt but the ocean is changing so fast.” </p>
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="d5aaf60e3a427f278747acf0c3e7ba39b39ef923" id="img-13">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1d00d3ea91d32110b40d85ff43227728"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3200"/>
<a href="#img-13" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2a64f318f02a31bdf9f17ae01fca72a3 1760w" />
<source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=811b3a2ecee1a87b72813d328f59aa85 880w" />
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=47418e3f630c00af6124c471fdc54c20 1600w" />
<source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=93ec03888887d5eb6eabf07e40f2ed3b 800w" />
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=476d165704890f40cc5f2d74f4d09b95 1280w" />
<source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6179da6bca697f83ac063a56660aa31b 640w" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=df0c5b9ccac02dcd010d7f3ed8ff8c85 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e9fd2eedc9bb80286a6cabd1c7c3b0e2 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e48f277772be7e455f9a4a4139b3bf4c 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2e0af911fcd7011f04ee91d445290e84 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=15161e07957f327b44aa124d94ae8291 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ef6c6809f2adcff0f4ff32899095839b 445w" />
<img itemprop="contentUrl" alt="The skull of a Brydes whale, in the storage container at Hihiaua Cultural Centre, Whangārei." src="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1b74b488aedb864287ff160f86d74c9d" />
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)"
sizes="880px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2a64f318f02a31bdf9f17ae01fca72a3 1760w"/>
<source media="(min-width: 1300px)" sizes="880px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=811b3a2ecee1a87b72813d328f59aa85 880w"/>
<source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)"
sizes="800px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=47418e3f630c00af6124c471fdc54c20 1600w"/>
<source media="(min-width: 1140px)" sizes="800px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=93ec03888887d5eb6eabf07e40f2ed3b 800w"/>
<source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)"
sizes="640px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=476d165704890f40cc5f2d74f4d09b95 1280w"/>
<source media="(min-width: 980px)" sizes="640px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6179da6bca697f83ac063a56660aa31b 640w"/>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=df0c5b9ccac02dcd010d7f3ed8ff8c85 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e9fd2eedc9bb80286a6cabd1c7c3b0e2 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e48f277772be7e455f9a4a4139b3bf4c 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2e0af911fcd7011f04ee91d445290e84 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=15161e07957f327b44aa124d94ae8291 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ef6c6809f2adcff0f4ff32899095839b 445w"/>
<img itemprop="contentUrl"
alt="The skull of a Brydes whale, in the storage container at Hihiaua Cultural Centre, Whangārei."
src="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1b74b488aedb864287ff160f86d74c9d"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<ul>
<li>
<p> Above, the skull of a bryde’s whale; right, a large-calibre bullet of the type that the New Zealand Department of Conservation uses for euthanasing stranded whales that are beyond rescue </p>
<p> Above, the skull of a bryde’s whale; right, a large-calibre bullet of the type
that the New Zealand Department of Conservation uses for euthanasing stranded
whales that are beyond rescue </p>
</li>
</ul>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="3766106f73e858d5b140ae3cdd2eef84060180cd" id="img-14">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7dca3d798aef3526f42e281771d50f91" />
<meta itemprop="width" content="4800" />
<meta itemprop="height" content="3200" /><a href="#img-14" data-link-name="Launch Article Lightbox" data-is-ajax="">
<figure itemprop="associatedMedia image" itemscope="itemscope"
itemtype="http://schema.org/ImageObject" data-component="image"
data-media-id="3766106f73e858d5b140ae3cdd2eef84060180cd" id="img-14">
<meta itemprop="url"
content="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7dca3d798aef3526f42e281771d50f91"/>
<meta itemprop="width" content="4800"/>
<meta itemprop="height" content="3200"/>
<a href="#img-14" data-link-name="Launch Article Lightbox" data-is-ajax="">
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0daf794415132fc19259f8d2f654f57f 1240w" />
<source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98ce62f9d983fe8059c936a6c6cdff33 620w" />
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9ff4aac2a3b07867dfdfb8f470ea6977 1210w" />
<source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=10d1b71094ecf1722f593eb49bb2effe 605w" />
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0705690fcb523f2781a5952f83528ff9 890w" />
<source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4c748337df5f0348eb0e7d3e3ec46571 445w" />
<img itemprop="contentUrl" alt="A large calibre bullet of the type that the New Zealand Department of Conservation (DOC) uses for euthanasing stranded whales that are beyond rescue." src="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2f5bb7c3c3642ac8733ca40509f6e20" />
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)"
sizes="620px"
srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0daf794415132fc19259f8d2f654f57f 1240w"/>
<source media="(min-width: 660px)" sizes="620px"
srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98ce62f9d983fe8059c936a6c6cdff33 620w"/>
<source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)"
sizes="605px"
srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9ff4aac2a3b07867dfdfb8f470ea6977 1210w"/>
<source media="(min-width: 480px)" sizes="605px"
srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=10d1b71094ecf1722f593eb49bb2effe 605w"/>
<source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)"
sizes="445px"
srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0705690fcb523f2781a5952f83528ff9 890w"/>
<source media="(min-width: 0px)" sizes="445px"
srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4c748337df5f0348eb0e7d3e3ec46571 445w"/>
<img itemprop="contentUrl"
alt="A large calibre bullet of the type that the New Zealand Department of Conservation (DOC) uses for euthanasing stranded whales that are beyond rescue."
src="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2f5bb7c3c3642ac8733ca40509f6e20"/>
</picture>
</div><span><svg width="22" height="22" viewbox="0 0 22 22">
</div>
<span><svg width="22" height="22" viewbox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6"></path>
</svg></span>
</a>
</figure>
<p> The recent spate of mass strandings has been described as “heartbreaking” by the conservation department. </p>
<p> But for Parata and his family the slow, painful deaths of their ancestors are personal – and ultimately devastating – for the health of the tribe and the sea. </p>
<p> “It’s very emotional. Our ancestors tell us the strandings are a sign from the sea. So what is the sea telling us? We need to listen.” </p>
<p> The recent spate of mass strandings has been described as “heartbreaking” by the
conservation department. </p>
<p> But for Parata and his family the slow, painful deaths of their ancestors are personal –
and ultimately devastating – for the health of the tribe and the sea. </p>
<p> “It’s very emotional. Our ancestors tell us the strandings are a sign from the sea. So
what is the sea telling us? We need to listen.” </p>
</div>
</div>

3923
app/src/main/assets/readability/test/test-pages/guardian-1/source.html
View File

File diff suppressed because one or more lines are too long

41
app/src/main/assets/readability/test/test-pages/heise/expected.html
View File

@ -1,18 +1,45 @@
<div id="readability-page-1" class="page">
<div>
<figure>
<img src="http://3.f.ix.de/scale/geometry/600/q75/imgs/18/1/4/6/2/3/5/1/Barcode-Scanner-With-Border-fc08c913da5cea5d.jpeg" />
<img src="http://3.f.ix.de/scale/geometry/600/q75/imgs/18/1/4/6/2/3/5/1/Barcode-Scanner-With-Border-fc08c913da5cea5d.jpeg"/>
<figcaption>
<p class="caption">1Password scannt auch QR-Codes.</p>
<p>(Bild: Hersteller)</p>
</figcaption>
</figure>
<p><strong>Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X 10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere Verbesserungen.</strong></p>
<p><a rel="external" target="_blank" href="https://itunes.apple.com/de/app/1password-password-manager/id443987910">AgileBits hat Version 5.3 seines bekannten Passwortmanagers 1Password für OS X freigegeben.</a> Mit dem Update wird eine praktische Funktion nachgereicht, die <a href="http://fakehost/mac-and-i/meldung/Passwortmanager-1Password-mit-groesseren-Updates-fuer-OS-X-und-iOS-2529204.html">die iOS-Version der Anwendung bereits seit längerem beherrscht</a>: Das direkte Erstellen von Einmal-Passwörtern. Unterstützt wird dabei der <a rel="external" target="_blank" href="https://blog.agilebits.com/2015/01/26/totp-for-1password-users/">TOTP-Standard</a> (Time-Based One-Time Passwords), den unter anderem Firmen wie Evernote, Dropbox oder Google einsetzen, um ihre Zugänge besser abzusichern. Neben Account und regulärem Passwort wird dabei dann ein Zusatzcode verlangt, der nur kurze Zeit gilt.</p>
<p>Zur TOTP-Nutzung muss zunächst ein Startwert an 1Password übergeben werden. Das geht unter anderem per QR-Code, den die App über ein neues Scanfenster selbst einlesen kann – etwa aus dem Webbrowser. Eine Einführung in die Technik gibt <a rel="external" target="_blank" href="http://1pw.ca/TOTPvideoMac">ein kurzes Video</a>. Die TOTP-Unterstützung in 1Password erlaubt es, auf ein zusätzliches Gerät (z.B. ein iPhone) neben dem Mac zu verzichten, das den Code liefert – was allerdings auch die Sicherheit verringert, weil es keinen "echten" zweiten Faktor mehr gibt.</p>
<p>Update 5.3 des Passwortmanagers liefert auch noch weitere Verbesserungen. So gibt es die Möglichkeit, FaceTime-Audio- oder Skype-Anrufe aus 1Password zu starten, die Zahl der Zusatzfelder in der Datenbank wurde erweitert und der Umgang mit unterschiedlichen Zeitzonen klappt besser. Die Engine zur Passworteingabe im Browser soll beschleunigt worden sein.</p>
<p>1Password kostet aktuell knapp 50 Euro im Mac App Store und setzt in seiner aktuellen Version mindestens OS X 10.10 voraus. <span>(<a title="Ben Schwan" href="mailto:bsc@heise.de">bsc</a>)</span>
<br />
<p><strong>Das in der iOS-Version bereits enthaltene TOTP-Feature ist nun auch für OS X
10.10 verfügbar. Zudem gibt es neue Zusatzfelder in der Datenbank und weitere
Verbesserungen.</strong></p>
<p><a rel="external" target="_blank"
href="https://itunes.apple.com/de/app/1password-password-manager/id443987910">AgileBits
hat Version 5.3 seines bekannten Passwortmanagers 1Password für OS X freigegeben.</a>
Mit dem Update wird eine praktische Funktion nachgereicht, die <a
href="http://fakehost/mac-and-i/meldung/Passwortmanager-1Password-mit-groesseren-Updates-fuer-OS-X-und-iOS-2529204.html">die
iOS-Version der Anwendung bereits seit längerem beherrscht</a>: Das direkte
Erstellen von Einmal-Passwörtern. Unterstützt wird dabei der <a rel="external"
target="_blank"
href="https://blog.agilebits.com/2015/01/26/totp-for-1password-users/">TOTP-Standard</a>
(Time-Based One-Time Passwords), den unter anderem Firmen wie Evernote, Dropbox oder
Google einsetzen, um ihre Zugänge besser abzusichern. Neben Account und regulärem
Passwort wird dabei dann ein Zusatzcode verlangt, der nur kurze Zeit gilt.</p>
<p>Zur TOTP-Nutzung muss zunächst ein Startwert an 1Password übergeben werden. Das geht
unter anderem per QR-Code, den die App über ein neues Scanfenster selbst einlesen kann –
etwa aus dem Webbrowser. Eine Einführung in die Technik gibt <a rel="external"
target="_blank"
href="http://1pw.ca/TOTPvideoMac">ein
kurzes Video</a>. Die TOTP-Unterstützung in 1Password erlaubt es, auf ein
zusätzliches Gerät (z.B. ein iPhone) neben dem Mac zu verzichten, das den Code liefert –
was allerdings auch die Sicherheit verringert, weil es keinen "echten" zweiten Faktor
mehr gibt.</p>
<p>Update 5.3 des Passwortmanagers liefert auch noch weitere Verbesserungen. So gibt es die
Möglichkeit, FaceTime-Audio- oder Skype-Anrufe aus 1Password zu starten, die Zahl der
Zusatzfelder in der Datenbank wurde erweitert und der Umgang mit unterschiedlichen
Zeitzonen klappt besser. Die Engine zur Passworteingabe im Browser soll beschleunigt
worden sein.</p>
<p>1Password kostet aktuell knapp 50 Euro im Mac App Store und setzt in seiner aktuellen
Version mindestens OS X 10.10 voraus. <span>(<a title="Ben Schwan"
href="mailto:bsc@heise.de">bsc</a>)</span>
<br/>
</p>
</div>
</div>

1394
app/src/main/assets/readability/test/test-pages/heise/source.html
View File

File diff suppressed because it is too large Load Diff

87
app/src/main/assets/readability/test/test-pages/herald-sun-1/expected.html
View File

@ -1,32 +1,83 @@
<div id="readability-page-1" class="page">
<div>
<div>
<p><img data-src="http://api.news.com.au/content/1.0/heraldsun/images/1227261885862?format=jpg&amp;group=iphone&amp;size=medium" alt="A new Bill would require telecommunications service providers to store so-called ‘metadat" />
<p>
<img data-src="http://api.news.com.au/content/1.0/heraldsun/images/1227261885862?format=jpg&amp;group=iphone&amp;size=medium"
alt="A new Bill would require telecommunications service providers to store so-called ‘metadat"/>
</p>
<p class="caption">
<span id="imgCaption">A new Bill would require telecommunications service providers to store so-called ‘metadata’ for two years.</span>
<span><em>Source:</em> Supplied</span>
</p>
</div>
<p><strong> A HIGH-powered federal government team has been doing the rounds of media organisations in the past few days in an attempt to allay concerns about the impact of new surveillance legislation on press freedom. It failed. </strong></p>
<p>The roadshow featured the Prime Minister’s national security adviser, Andrew Shearer, Justin Bassi, who advises Attorney-General George Brandis on crime and security matters, and Australian Federal Police Commissioner Andrew Colvin. Staffers from the office of Communications Minister Malcolm Turnbull also took part.</p>
<p>They held meetings with executives from News Corporation and Fairfax, representatives of the TV networks, the ABC top brass and a group from the media union and the Walkley journalism foundation. I was involved as a member of the Walkley board.</p>
<p>The initiative, from Tony Abbott’s office, is evidence that the Government has been alarmed by the strength of criticism from media of the Data Retention Bill it wants passed before Parliament rises in a fortnight. Bosses, journalists, even the Press Council, are up in arms, not only over this measure, but also over aspects of two earlier pieces of national security legislation that interfere with the ability of the media to hold government to account.</p>
<p><strong> A HIGH-powered federal government team has been doing the rounds of media
organisations in the past few days in an attempt to allay concerns about the impact of
new surveillance legislation on press freedom. It failed. </strong></p>
<p>The roadshow featured the Prime Minister’s national security adviser, Andrew Shearer,
Justin Bassi, who advises Attorney-General George Brandis on crime and security matters,
and Australian Federal Police Commissioner Andrew Colvin. Staffers from the office of
Communications Minister Malcolm Turnbull also took part.</p>
<p>They held meetings with executives from News Corporation and Fairfax, representatives of
the TV networks, the ABC top brass and a group from the media union and the Walkley
journalism foundation. I was involved as a member of the Walkley board.</p>
<p>The initiative, from Tony Abbott’s office, is evidence that the Government has been
alarmed by the strength of criticism from media of the Data Retention Bill it wants
passed before Parliament rises in a fortnight. Bosses, journalists, even the Press
Council, are up in arms, not only over this measure, but also over aspects of two
earlier pieces of national security legislation that interfere with the ability of the
media to hold government to account.</p>
<div id="read-more">
<p>The Bill would require telecommunications service providers to store so-called “metadata” — the who, where, when and how of a communication, but not its content — for two years so security and law enforcement agencies can access it without warrant. Few would argue against the use of such material to catch criminals or terrorists. But, as Parliament’s Joint Committee on Intelligence and Security has pointed out, it would also be used “for the purpose of determining the identity of a journalist’s sources”.</p>
<p>And that should ring warning bells for anyone genuinely concerned with the health of our democracy. Without the ability to protect the identity of sources, journalists would be greatly handicapped in exposing corruption, dishonesty, waste, incompetence and misbehaviour by public officials.</p>
<p>The Bill would require telecommunications service providers to store so-called
“metadata” — the who, where, when and how of a communication, but not its content —
for two years so security and law enforcement agencies can access it without
warrant. Few would argue against the use of such material to catch criminals or
terrorists. But, as Parliament’s Joint Committee on Intelligence and Security has
pointed out, it would also be used “for the purpose of determining the identity of a
journalist’s sources”.</p>
<p>And that should ring warning bells for anyone genuinely concerned with the health of
our democracy. Without the ability to protect the identity of sources, journalists
would be greatly handicapped in exposing corruption, dishonesty, waste, incompetence
and misbehaviour by public officials.</p>
<p>The Press Council is concerned the laws would crush investigative journalism.</p>
<p>“These legitimate concerns cannot be addressed effectively short of exempting journalists and media organisations,” says president David Weisbrot.</p>
<p>The media union is adamant journalists’ metadata must be exempted from the law. That’s what media bosses want, too, though they have a fallback position based on new safeguards being implemented in Britain.</p>
<p>That would prevent access to the metadata of journalists or media organisations without a judicial warrant. There would be a code including — according to the explanatory notes of the British Bill — “provision to protect the public interest in the confidentiality of journalistic sources”.</p>
<p>In their meetings this week, the government team boasted of concessions in the new Data Retention Bill. The number of agencies able to access metadata will be reduced by excluding such organisations as the RSPCA and local councils. And whenever an authorisation is issued for access to information about a journalist’s sources, the Ombudsman (or, where ASIO is involved, the Inspector-General of Intelligence and Security) will receive a copy.</p>
<p>That does nothing to solve the problem. The Government has effectively admitted as much by agreeing that the parliamentary committee should conduct a separate review of how to deal with the issue of journalists’ sources.</p>
<p>But another inquiry would be a waste of time — the committee has already received and considered dozens of submissions on the subject. The bottom line is that the Government does not deny that the legislation is flawed, but is demanding it be passed anyway with the possibility left open of a repair job down the track. That is a ridiculous approach.</p>
<p>Claims that immediate action is imperative do not stand up. These are measures that won’t come into full effect for two years. Anyway, amending the Bill to either exempt journalists or adopt the UK model could be done quickly, without any risk to national security.</p>
<p>AS Opposition Leader Bill Shorten said in a letter to Abbott last month: “Press freedom concerns about mandatory data retention would ideally be addressed in this Bill to avoid the need for future additional amendments or procedures to be put in place in the future.”</p>
<p>The Data Retention Bill will be debated in the House of Representatives this week. Then, on Friday, CEOs from leading media organisations will front the parliamentary committee to air their concerns before the legislation goes to the Senate.</p>
<p>Those CEOs should make it clear they are just as angry about this as they were about Stephen Conroy’s attempt to impinge on press freedom through media regulation under the previous Labor government.</p>
<p>Memories of the grief Conroy brought down on his head would undoubtedly make Abbott sit up and take notice.</p>
<p>“These legitimate concerns cannot be addressed effectively short of exempting
journalists and media organisations,” says president David Weisbrot.</p>
<p>The media union is adamant journalists’ metadata must be exempted from the law.
That’s what media bosses want, too, though they have a fallback position based on
new safeguards being implemented in Britain.</p>
<p>That would prevent access to the metadata of journalists or media organisations
without a judicial warrant. There would be a code including — according to the
explanatory notes of the British Bill — “provision to protect the public interest in
the confidentiality of journalistic sources”.</p>
<p>In their meetings this week, the government team boasted of concessions in the new
Data Retention Bill. The number of agencies able to access metadata will be reduced
by excluding such organisations as the RSPCA and local councils. And whenever an
authorisation is issued for access to information about a journalist’s sources, the
Ombudsman (or, where ASIO is involved, the Inspector-General of Intelligence and
Security) will receive a copy.</p>
<p>That does nothing to solve the problem. The Government has effectively admitted as
much by agreeing that the parliamentary committee should conduct a separate review
of how to deal with the issue of journalists’ sources.</p>
<p>But another inquiry would be a waste of time — the committee has already received and
considered dozens of submissions on the subject. The bottom line is that the
Government does not deny that the legislation is flawed, but is demanding it be
passed anyway with the possibility left open of a repair job down the track. That is
a ridiculous approach.</p>
<p>Claims that immediate action is imperative do not stand up. These are measures that
won’t come into full effect for two years. Anyway, amending the Bill to either
exempt journalists or adopt the UK model could be done quickly, without any risk to
national security.</p>
<p>AS Opposition Leader Bill Shorten said in a letter to Abbott last month: “Press
freedom concerns about mandatory data retention would ideally be addressed in this
Bill to avoid the need for future additional amendments or procedures to be put in
place in the future.”</p>
<p>The Data Retention Bill will be debated in the House of Representatives this week.
Then, on Friday, CEOs from leading media organisations will front the parliamentary
committee to air their concerns before the legislation goes to the Senate.</p>
<p>Those CEOs should make it clear they are just as angry about this as they were about
Stephen Conroy’s attempt to impinge on press freedom through media regulation under
the previous Labor government.</p>
<p>Memories of the grief Conroy brought down on his head would undoubtedly make Abbott
sit up and take notice.</p>
<p><b>LAURIE OAKES IS THE NINE NETWORK POLITICAL EDITOR </b></p>
</div>
</div>

2096
app/src/main/assets/readability/test/test-pages/herald-sun-1/source.html
View File

File diff suppressed because it is too large Load Diff

12
app/src/main/assets/readability/test/test-pages/hidden-nodes/expected.html
View File

@ -1,5 +1,15 @@
<div id="readability-page-1" class="page">
<p> Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. </p>
<p> Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam
et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est
Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed
diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd
gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit
amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et
dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores
et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit
amet. </p>
<h2> Secondary header </h2>
<h2> Third header </h2>
</div>

80
app/src/main/assets/readability/test/test-pages/hidden-nodes/source.html
View File

@ -1,29 +1,57 @@
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta charset="utf-8" />
<title>
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
</title>
</head>
<body>
<h1>
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
</h1>
<p>
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
</p>
<h2>
Secondary header
</h2>
<p style="display: none;">
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
</p>
<h2>
Third header
</h2>
<p hidden="hidden">
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
</p>
</body>
<head>
<meta charset="utf-8"/>
<title>
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
invidunt
</title>
</head>
<body>
<h1>
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
</h1>
<p>
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo
dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor
sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et
justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem
ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos
et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus
est Lorem ipsum dolor sit amet.
</p>
<h2>
Secondary header
</h2>
<p style="display: none;">
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo
dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor
sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et
justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem
ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos
et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus
est Lorem ipsum dolor sit amet.
</p>
<h2>
Third header
</h2>
<p hidden="hidden">
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt
ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo
dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor
sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor
invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et
justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem
ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos
et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus
est Lorem ipsum dolor sit amet.
</p>
</body>
</html>

373
app/src/main/assets/readability/test/test-pages/hukumusume/expected.html
View File

@ -3,244 +3,291 @@
<td>
<table>
<tbody>
<tr>
<td>
<img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7" />
</td>
<td></td>
<td>
<img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7" />
</td>
</tr>
<tr>
<td>
<img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7"/>
</td>
<td></td>
<td>
<img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7"/>
</td>
</tr>
</tbody>
</table>
<table>
<tbody>
<tr>
<td>
</td>
</tr>
<tr>
<td> &#160; </td>
</tr>
<tr>
<td>
</td>
</tr>
<tr>
<td> &#160;</td>
</tr>
</tbody>
</table>
</td>
<td>
<p>
<a href="http://fakehost/index.html">福娘童話集</a> &gt; <a href="http://fakehost/index.html">きょうのイソップ童話</a> &gt; <a href="http://fakehost/itiran/01gatu.htm">１月のイソップ童話</a> &gt; 欲張りなイヌ
<a href="http://fakehost/index.html">福娘童話集</a> &gt; <a
href="http://fakehost/index.html">きょうのイソップ童話</a> &gt; <a
href="http://fakehost/itiran/01gatu.htm">１月のイソップ童話</a> &gt; 欲張りなイヌ
</p>
<div>
<p><span color="#FF0000" size="+2">元旦のイソップ童話</span></p>
<p>
<img src="http://fakehost/gazou/pc_gazou/aesop/aesop052.jpg" alt="よくばりなイヌ" width="480" height="360" />
<img src="http://fakehost/gazou/pc_gazou/aesop/aesop052.jpg" alt="よくばりなイヌ"
width="480" height="360"/>
</p>
<p> 欲張りなイヌ</p>
<p>
<a href="http://hukumusume.com/douwa/English/aesop/01/01_j.html">ひらがな</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_j&amp;E.html">日本語・英語</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_E.html">English</a>
<a href="http://hukumusume.com/douwa/English/aesop/01/01_j.html">ひらがな</a>　←→　<a
href="http://hukumusume.com/douwa/English/aesop/01/01_j&amp;E.html">日本語・英語</a>　←→　<a
href="http://hukumusume.com/douwa/English/aesop/01/01_E.html">English</a>
</p>
</div>
<div>
<table>
<tbody>
<tr>
<td>
<img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7" />
</td>
<td>
<span color="#FF0000"><b>おりがみをつくろう</b></span>
</td>
<td>
<span size="-1">( <a href="http://www.origami-club.com/index.html">おりがみくらぶ</a> より)</span>
</td>
<td>
<img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7" />
</td>
</tr>
<tr>
<td colspan="4">
<p>
<a href="http://www.origami-club.com/easy/dogfase/index.html"><span size="+2"><img src="http://fakehost/gazou/origami_gazou/kantan/dogface.gif" alt="犬の顔の折り紙" width="73" height="51" />いぬのかお</span></a>　　　<a href="http://www.origami-club.com/easy/dog/index.html"><img src="http://fakehost/gazou/origami_gazou/kantan/dog.gif" alt="犬の顔の紙" width="62" height="43" /><span size="+2">いぬ</span></a>
</p>
</td>
</tr>
<tr>
<td>
<img src="http://fakehost/366/logo_bana/corner_1.gif" width="7"
height="7"/>
</td>
<td>
<span color="#FF0000"><b>おりがみをつくろう</b></span>
</td>
<td>
<span size="-1">( <a href="http://www.origami-club.com/index.html">おりがみくらぶ</a> より)</span>
</td>
<td>
<img src="http://fakehost/366/logo_bana/corner_2.gif" width="7"
height="7"/>
</td>
</tr>
<tr>
<td colspan="4">
<p>
<a href="http://www.origami-club.com/easy/dogfase/index.html"><span
size="+2"><img
src="http://fakehost/gazou/origami_gazou/kantan/dogface.gif"
alt="犬の顔の折り紙" width="73" height="51"/>いぬのかお</span></a>　　　<a
href="http://www.origami-club.com/easy/dog/index.html"><img
src="http://fakehost/gazou/origami_gazou/kantan/dog.gif"
alt="犬の顔の紙" width="62" height="43"/><span
size="+2">いぬ</span></a>
</p>
</td>
</tr>
</tbody>
</table>
</div>
<table>
<tbody>
<tr>
<td> ♪音声配信(html5) </td>
</tr>
<tr>
<td>
<audio src="http://ohanashi2.up.seesaa.net/mp3/ae_0101.mp3" controls=""></audio>
</td>
</tr>
<tr>
<td>
<a href="http://www.voiceblog.jp/onokuboaki/"><span size="-1">亜姫の朗読☆　イソップ童話より</span></a>
</td>
</tr>
<tr>
<td> ♪音声配信(html5)</td>
</tr>
<tr>
<td>
<audio src="http://ohanashi2.up.seesaa.net/mp3/ae_0101.mp3"
controls=""></audio>
</td>
</tr>
<tr>
<td>
<a href="http://www.voiceblog.jp/onokuboaki/"><span size="-1">亜姫の朗読☆　イソップ童話より</span></a>
</td>
</tr>
</tbody>
</table>
<p> 　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ） 　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。 「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。 　さっきの川の中のイヌは、水にうつった自分の顔だったのです。 　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。 </p>
<p> 　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ）
　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。
「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。
　さっきの川の中のイヌは、水にうつった自分の顔だったのです。
　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。 </p>
<p> おしまい </p>
<div>
<p><span><img src="http://fakehost/gazou/pc_gazou/all/top_bana/back_logo_r.gif" alt="前のページへ戻る" name="Image10" width="175" height="32" id="Image10" /></span></p>
<p><span><img src="http://fakehost/gazou/pc_gazou/all/top_bana/back_logo_r.gif"
alt="前のページへ戻る" name="Image10" width="175" height="32"
id="Image10"/></span></p>
</div>
</td>
<td>
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" />
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/>
</td>
<td>
<table>
<tbody>
<tr>
<td>
<img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7" />
</td>
<td></td>
<td>
<img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7" />
</td>
</tr>
<tr>
<td>
<img src="http://fakehost/366/logo_bana/corner_1.gif" width="7" height="7"/>
</td>
<td></td>
<td>
<img src="http://fakehost/366/logo_bana/corner_2.gif" width="7" height="7"/>
</td>
</tr>
</tbody>
</table>
<table>
<tbody>
<tr>
<td> &#160;&#160;&#160;&#160; <span size="-1"><b>1月 1日の豆知識</b></span><span size="-2"><u>
<tr>
<td> &#160;&#160;&#160;&#160; <span size="-1"><b>1月 1日の豆知識</b></span><span
size="-2"><u>
<p> 366日への旅</p>
</u></span>
</td>
</tr>
<tr>
<td>
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97" width="1" height="1" /><b><span size="-1">きょうの記念日</span></b><a href="http://fakehost/366/kinenbi/pc/01gatu/1_01.htm"><span size="-1">元旦</span></a>
</td>
</tr>
<tr>
<td>
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><span size="-1">きょうの誕生花</span></b><a href="http://fakehost/366/hana/pc/01gatu/1_01.htm"><span size="-1">松(まつ)</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの誕生日・出来事</span></b><a href="http://fakehost/366/birthday/pc/01gatu/1_01.htm"><span size="-1">1949年　Mr.マリック(マジシャン)</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">恋の誕生日占い</span></b><a href="http://fakehost/sakura/uranai/birthday/01/01.html"><span size="-1">自分の考えをしっかりと持った女の子。</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">なぞなぞ小学校</span></b><a href="http://fakehost/nazonazo/new/2012/04/02.html"><span size="-1">○(丸)を取ったらお母さんになってしまう男の人は？</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">あこがれの職業紹介</span></b><a href="http://fakehost/sakura/navi/work/2017/041.html"><span size="-1">歌手</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">恋の魔法とおまじない</span></b> 001<a href="http://fakehost/omajinai/new/2012/00/re01.html"><span size="-1">両思いになれる おまじない</span></a>
</td>
</tr>
<tr>
<td>
</td>
</tr>
<tr>
<td>
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97"
width="1" height="1"/><b><span size="-1">きょうの記念日</span></b><a
href="http://fakehost/366/kinenbi/pc/01gatu/1_01.htm"><span
size="-1">元旦</span></a>
</td>
</tr>
<tr>
<td>
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/><b><span size="-1">きょうの誕生花</span></b><a
href="http://fakehost/366/hana/pc/01gatu/1_01.htm"><span
size="-1">松(まつ)</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの誕生日・出来事</span></b><a
href="http://fakehost/366/birthday/pc/01gatu/1_01.htm"><span size="-1">1949年　Mr.マリック(マジシャン)</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">恋の誕生日占い</span></b><a
href="http://fakehost/sakura/uranai/birthday/01/01.html"><span
size="-1">自分の考えをしっかりと持った女の子。</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">なぞなぞ小学校</span></b><a
href="http://fakehost/nazonazo/new/2012/04/02.html"><span size="-1">○(丸)を取ったらお母さんになってしまう男の人は？</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">あこがれの職業紹介</span></b><a
href="http://fakehost/sakura/navi/work/2017/041.html"><span
size="-1">歌手</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">恋の魔法とおまじない</span></b> 001<a
href="http://fakehost/omajinai/new/2012/00/re01.html"><span size="-1">両思いになれる おまじない</span></a>
</td>
</tr>
<tr>
<td>
<span size="-1">　　<b>1月 1日の童話・昔話</b><u><span size="-2">
<p> 福娘童話集</p>
</span></u></span>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの日本昔話</span></b><a href="http://fakehost/douwa/pc/jap/01/01.htm"><span size="-1">ネコがネズミを追いかける訳</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの世界昔話<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /></span></b><a href="http://fakehost/douwa/pc/world/01/01a.htm"><span size="-1">モンゴルの十二支話</span></a>
</td>
</tr>
<tr>
<td>
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><span size="-1">きょうの日本民話</span></b><a href="http://fakehost/douwa/pc/minwa/01/01c.html"><span size="-1">仕事の取替えっこ</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうのイソップ童話</span></b><a href="http://fakehost/douwa/pc/aesop/01/01.htm"><span size="-1">欲張りなイヌ</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの江戸小話</span></b><a href="http://fakehost/douwa/pc/kobanashi/01/01.htm"><span size="-1">ぞうきんとお年玉</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの百物語</span></b><a href="http://fakehost/douwa/pc/kaidan/01/01.htm"><span size="-1">百物語の幽霊</span></a>
</td>
</tr>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの日本昔話</span></b><a
href="http://fakehost/douwa/pc/jap/01/01.htm"><span size="-1">ネコがネズミを追いかける訳</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの世界昔話<img
src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/></span></b><a
href="http://fakehost/douwa/pc/world/01/01a.htm"><span size="-1">モンゴルの十二支話</span></a>
</td>
</tr>
<tr>
<td>
<img src="file:///C:/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/><b><span size="-1">きょうの日本民話</span></b><a
href="http://fakehost/douwa/pc/minwa/01/01c.html"><span size="-1">仕事の取替えっこ</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうのイソップ童話</span></b><a
href="http://fakehost/douwa/pc/aesop/01/01.htm"><span
size="-1">欲張りなイヌ</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの江戸小話</span></b><a
href="http://fakehost/douwa/pc/kobanashi/01/01.htm"><span size="-1">ぞうきんとお年玉</span></a>
</td>
</tr>
<tr>
<td>
<b><span size="-1">きょうの百物語</span></b><a
href="http://fakehost/douwa/pc/kaidan/01/01.htm"><span
size="-1">百物語の幽霊</span></a>
</td>
</tr>
</tbody>
</table>
<table>
<tbody>
<tr>
<td>
<b><span size="-1">福娘のサイト</span></b>
</td>
</tr>
<tr>
<td>
<tr>
<td>
<b><span size="-1">福娘のサイト</span></b>
</td>
</tr>
<tr>
<td>
<span size="-1"><b>366日への旅</b>
<p>
<a href="http://hukumusume.com/366/">毎日の記念日・誕生花 ・有名人の誕生日と性格判断</a>
</p>
</span>
</td>
</tr>
<tr>
<td>
</td>
</tr>
<tr>
<td>
<span size="-1"><b>福娘童話集</b>
<p>
<a href="http://hukumusume.com/douwa/">世界と日本の童話と昔話</a>
</p>
</span>
</td>
</tr>
<tr>
<td>
</td>
</tr>
<tr>
<td>
<span size="-1"><b>女の子応援サイト -さくら-</b>
<p>
<a href="http://hukumusume.com/sakura/index.html">誕生日占い、お仕事紹介、おまじない、など</a>
</p>
</span>
</td>
</tr>
<tr>
<td>
</td>
</tr>
<tr>
<td>
<span size="-1"><b>子どもの病気相談所</b>
<p>
<a href="http://hukumusume.com/my_baby/sick/">病気検索と対応方法、症状から検索するWEB問診</a>
</p>
</span>
</td>
</tr>
<tr>
<td>
</td>
</tr>
<tr>
<td>
<span size="-1"><b>世界60秒巡り</b>
<p>
<a href="http://hukumusume.com/366/world/">国旗国歌や世界遺産など、世界の国々の豆知識</a>
</p>
</span>
</td>
</tr>
</td>
</tr>
</tbody>
</table>
</td>

696
app/src/main/assets/readability/test/test-pages/hukumusume/source.html
View File

@ -1,55 +1,59 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>
欲張りなイヌ　＜福娘童話集　きょうのイソップ童話＞
</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body bgcolor="#FFFFFF" text="#000000">
<table width="969" border="0" align="center" cellpadding="0" cellspacing="0">
<tbody>
<head>
<title>
欲張りなイヌ　＜福娘童話集　きょうのイソップ童話＞
</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
</head>
<body bgcolor="#FFFFFF" text="#000000">
<table width="969" border="0" align="center" cellpadding="0" cellspacing="0">
<tbody>
<tr>
<td colspan="5" height="12">
<div align="center">
<table width="100%" border="0">
<tbody>
<tr>
<td width="27%" align="center">
<a href="../../../index.html">福娘童話集</a> &gt; <a href="../index.html">きょうのイソップ童話</a>
</td>
<td width="46%" align="center">
<a href="http://hukumusume.com/douwa/pc/aesop/index.html"><img
src="../../../gazou/pc_gazou/all/aesop_logo_llll.gif"
alt="福娘童話集　きょうのイソップ童話" width="320" height="100" border="0"/></a>
</td>
<td width="27%" align="center" valign="bottom">
<a href="http://hukumusume.com/douwa/index.html"><img
src="../../../gazou/pc_gazou/all/douwa_logo_top_.gif"
alt="童話・昔話・おとぎ話の福娘童話集" width="170" height="50" border="0"/></a>
</td>
</tr>
</tbody>
</table>
</div>
</td>
</tr>
<tr>
<td width="166" height="830" valign="top">
<table width="166" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td colspan="5" height="12">
<div align="center">
<table width="100%" border="0">
<tbody>
<tr>
<td width="27%" align="center">
<a href="../../../index.html">福娘童話集</a> &gt; <a href="../index.html">きょうのイソップ童話</a>
</td>
<td width="46%" align="center">
<a href="http://hukumusume.com/douwa/pc/aesop/index.html"><img src="../../../gazou/pc_gazou/all/aesop_logo_llll.gif" alt="福娘童話集　きょうのイソップ童話" width="320" height="100" border="0" /></a>
</td>
<td width="27%" align="center" valign="bottom">
<a href="http://hukumusume.com/douwa/index.html"><img src="../../../gazou/pc_gazou/all/douwa_logo_top_.gif" alt="童話・昔話・おとぎ話の福娘童話集" width="170" height="50" border="0" /></a>
</td>
</tr>
</tbody>
</table>
</div>
<td height="7" valign="top">
<img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7"/>
</td>
<td></td>
<td align="right" valign="top">
<img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7"/>
</td>
</tr>
</tbody>
</table>
<table width="166" border="0" cellpadding="0" cellspacing="0">
<tbody>
<tr>
<td width="166" height="830" valign="top">
<table width="166" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td height="7" valign="top">
<img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7" />
</td>
<td></td>
<td align="right" valign="top">
<img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7" />
</td>
</tr>
</tbody>
</table>
<table width="166" border="0" cellpadding="0" cellspacing="0">
<tbody>
<tr>
<td width="310" height="610" align="center" bgcolor="#C8FFC8">
<script type="text/javascript">
<td width="310" height="610" align="center" bgcolor="#C8FFC8">
<script type="text/javascript">
//<![CDATA[
<!--
google_ad_client = "ca-pub-2746615155806331";
@ -59,107 +63,134 @@
google_ad_height = 600;
//-->
//]]>
</script>
<script type="text/javascript" src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></script>
</td>
</tr>
<tr>
<td align="center">
&#160;
</td>
</tr>
</script>
<script type="text/javascript"
src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></script>
</td>
</tr>
<tr>
<td align="center">
&#160;
</td>
</tr>
</tbody>
</table>
</td>
<td width="619" valign="top">
<p align="center">
<a href="../../../index.html">福娘童話集</a> &gt; <a href="../index.html">きょうのイソップ童話</a>
&gt; <a href="../itiran/01gatu.htm">１月のイソップ童話</a> &gt; 欲張りなイヌ
</p>
<p align="center">
<font color="#FF0000" size="+2">元旦のイソップ童話</font><br/>
<br/>
<br/>
<br/>
<img src="../../../gazou/pc_gazou/aesop/aesop052.jpg" alt="よくばりなイヌ" width="480"
height="360" border="1"/><br/>
<br/>
<br/>
<br/>
欲張りなイヌ<br/>
<br/>
<br/>
<br/>
<a href="http://hukumusume.com/douwa/English/aesop/01/01_j.html">ひらがな</a>　←→　<a
href="http://hukumusume.com/douwa/English/aesop/01/01_j&amp;E.html">日本語・英語</a>　←→　<a
href="http://hukumusume.com/douwa/English/aesop/01/01_E.html">English</a>
</p>
<table width="100%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td height="90" align="center">
<table width="80%" border="0" cellpadding="0" cellspacing="0"
bgcolor="#C8FFC8">
<tbody>
<tr>
<td width="35%" height="25" valign="top">
<img src="../../../../366/logo_bana/corner_1.gif" width="7"
height="7"/>
</td>
<td width="29%" align="center">
<font color="#FF0000"><b>おりがみをつくろう</b></font>
</td>
<td width="35%" valign="bottom">
<font size="-1">( <a
href="http://www.origami-club.com/index.html">おりがみくらぶ</a>
より)</font>
</td>
<td width="1%" align="right" valign="top">
<img src="../../../../366/logo_bana/corner_2.gif" width="7"
height="7"/>
</td>
</tr>
<tr>
<td height="80" colspan="4" align="center" valign="top">
<table width="98%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td height="75" align="center" valign="middle"
bgcolor="#ECFFEC">
<a href="http://www.origami-club.com/easy/dogfase/index.html"><font
size="+2"><img
src="../../../gazou/origami_gazou/kantan/dogface.gif"
alt="犬の顔の折り紙" width="73" height="51"
border="0"/>いぬのかお</font></a>　　　<a
href="http://www.origami-club.com/easy/dog/index.html"><img
src="../../../gazou/origami_gazou/kantan/dog.gif"
alt="犬の顔の紙" width="62" height="43"
border="0"/><font size="+2">いぬ</font></a>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
<td width="619" valign="top">
<p align="center">
<a href="../../../index.html">福娘童話集</a> &gt; <a href="../index.html">きょうのイソップ童話</a> &gt; <a href="../itiran/01gatu.htm">１月のイソップ童話</a> &gt; 欲張りなイヌ
</p>
<p align="center">
<font color="#FF0000" size="+2">元旦のイソップ童話</font><br />
<br />
<br />
<br />
<img src="../../../gazou/pc_gazou/aesop/aesop052.jpg" alt="よくばりなイヌ" width="480" height="360" border="1" /><br />
<br />
<br />
<br />
欲張りなイヌ<br />
<br />
<br />
<br />
<a href="http://hukumusume.com/douwa/English/aesop/01/01_j.html">ひらがな</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_j&amp;E.html">日本語・英語</a>　←→　<a href="http://hukumusume.com/douwa/English/aesop/01/01_E.html">English</a>
</p>
<table width="100%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td height="90" align="center">
<table width="80%" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td width="35%" height="25" valign="top">
<img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7" />
</td>
<td width="29%" align="center">
<font color="#FF0000"><b>おりがみをつくろう</b></font>
</td>
<td width="35%" valign="bottom">
<font size="-1">( <a href="http://www.origami-club.com/index.html">おりがみくらぶ</a> より)</font>
</td>
<td width="1%" align="right" valign="top">
<img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7" />
</td>
</tr>
<tr>
<td height="80" colspan="4" align="center" valign="top">
<table width="98%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td height="75" align="center" valign="middle" bgcolor="#ECFFEC">
<a href="http://www.origami-club.com/easy/dogfase/index.html"><font size="+2"><img src="../../../gazou/origami_gazou/kantan/dogface.gif" alt="犬の顔の折り紙" width="73" height="51" border="0" />いぬのかお</font></a>　　　<a href="http://www.origami-club.com/easy/dog/index.html"><img src="../../../gazou/origami_gazou/kantan/dog.gif" alt="犬の顔の紙" width="62" height="43" border="0" /><font size="+2">いぬ</font></a>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
<table width="100%" border="0">
<tbody>
<tr>
<td align="center">
♪音声配信(html5)
</td>
</tr>
<tr>
<td align="center">
<audio src="http://ohanashi2.up.seesaa.net/mp3/ae_0101.mp3" controls=""></audio>
</td>
</tr>
<tr>
<td align="center">
<a href="http://www.voiceblog.jp/onokuboaki/"><font size="-1">亜姫の朗読☆　イソップ童話より</font></a>
</td>
</tr>
</tbody>
</table>
<p>
　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ） 　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。 「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。 　さっきの川の中のイヌは、水にうつった自分の顔だったのです。 　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。
</p>
<p align="center">
おしまい
</p>
<p align="center">
<a href="javascript:history.back();" onmouseout="MM_swapImgRestore()" onmouseover="MM_swapImage('Image10','','../../../gazou/pc_gazou/all/top_bana/back_logo_b.gif',1)"><img src="../../../gazou/pc_gazou/all/top_bana/back_logo_r.gif" alt="前のページへ戻る" name="Image10" width="175" height="32" border="0" id="Image10" /></a><br />
<br />
<br />
<br />
<script type="text/javascript">
</tr>
</tbody>
</table>
<table width="100%" border="0">
<tbody>
<tr>
<td align="center">
♪音声配信(html5)
</td>
</tr>
<tr>
<td align="center">
<audio src="http://ohanashi2.up.seesaa.net/mp3/ae_0101.mp3"
controls=""></audio>
</td>
</tr>
<tr>
<td align="center">
<a href="http://www.voiceblog.jp/onokuboaki/"><font size="-1">亜姫の朗読☆　イソップ童話より</font></a>
</td>
</tr>
</tbody>
</table>
<p>
　肉をくわえたイヌが、橋を渡っていました。 　ふと下を見ると、川の中にも肉をくわえたイヌがいます。　イヌはそれを見て、思いました。（あいつの肉の方が、大きそうだ）
　イヌは、くやしくてたまりません。 （そうだ、あいつをおどかして、あの肉を取ってやろう） 　そこでイヌは、川の中のイヌに向かって思いっきり吠えました。
「ウゥー、ワン！！」 　そのとたん、くわえていた肉はポチャンと川の中に落ちてしまいました。 「ああー、ぁぁー」 　川の中には、がっかりしたイヌの顔がうつっています。
　さっきの川の中のイヌは、水にうつった自分の顔だったのです。 　同じ物を持っていても、人が持っている物の方が良く見え、また、欲張るとけっきょく損をするというお話しです。
</p>
<p align="center">
おしまい
</p>
<p align="center">
<a href="javascript:history.back();" onmouseout="MM_swapImgRestore()"
onmouseover="MM_swapImage('Image10','','../../../gazou/pc_gazou/all/top_bana/back_logo_b.gif',1)"><img
src="../../../gazou/pc_gazou/all/top_bana/back_logo_r.gif" alt="前のページへ戻る"
name="Image10" width="175" height="32" border="0" id="Image10"/></a><br/>
<br/>
<br/>
<br/>
<script type="text/javascript">
//<![CDATA[
<!--
@ -170,187 +201,204 @@
google_ad_height = 280;
//-->
//]]>
</script>
<script type="text/javascript" src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></script>
</p>
</script>
<script type="text/javascript"
src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></script>
</p>
</td>
<td width="10">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/>
</td>
<td width="166" valign="top">
<table width="100%" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td height="7" valign="top">
<img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7"/>
</td>
<td width="10">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" />
</td>
<td width="166" valign="top">
<table width="100%" border="0" cellpadding="0" cellspacing="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td height="7" valign="top">
<img src="../../../../366/logo_bana/corner_1.gif" width="7" height="7" />
</td>
<td></td>
<td align="right" valign="top">
<img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7" />
</td>
</tr>
</tbody>
</table>
<table width="166" border="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td width="156" height="50">
&#160;&#160;&#160;&#160; <font size="-1"><b>1月 1日の豆知識</b></font><br />
<br />
<font size="-2"><u><br />
<br />
366日への旅</u></font>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97" width="1" height="1" /><b><font size="-1">きょうの記念日</font></b><br />
<br />
<a href="../../../../366/kinenbi/pc/01gatu/1_01.htm"><font size="-1">元旦</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの誕生花</font></b><br />
<br />
<a href="../../../../366/hana/pc/01gatu/1_01.htm"><font size="-1">松(まつ)</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの誕生日・出来事</font></b><br />
<br />
<a href="../../../../366/birthday/pc/01gatu/1_01.htm"><font size="-1">1949年　Mr.マリック(マジシャン)</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">恋の誕生日占い</font></b><br />
<br />
<a href="../../../../sakura/uranai/birthday/01/01.html"><font size="-1">自分の考えをしっかりと持った女の子。</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">なぞなぞ小学校</font></b><br />
<br />
<a href="../../../../nazonazo/new/2012/04/02.html"><font size="-1">○(丸)を取ったらお母さんになってしまう男の人は？</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">あこがれの職業紹介</font></b><br />
<br />
<a href="../../../../sakura/navi/work/2017/041.html"><font size="-1">歌手</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">恋の魔法とおまじない</font></b> 001<br />
<br />
<a href="../../../../omajinai/new/2012/00/re01.html"><font size="-1">両思いになれる おまじない</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#C8FFC8">
<font size="-1">　　<b>1月 1日の童話・昔話</b><br />
<br />
<u><font size="-2"><br />
<br />
福娘童話集</font></u></font>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの日本昔話</font></b><br />
<br />
<a href="../../../../douwa/pc/jap/01/01.htm"><font size="-1">ネコがネズミを追いかける訳</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの世界昔話<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /></font></b><br />
<br />
<a href="../../../../douwa/pc/world/01/01a.htm"><font size="-1">モンゴルの十二支話</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif" width="1" height="1" /><b><font size="-1">きょうの日本民話</font></b><br />
<br />
<a href="../../../../douwa/pc/minwa/01/01c.html"><font size="-1">仕事の取替えっこ</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうのイソップ童話</font></b><br />
<br />
<a href="../../../../douwa/pc/aesop/01/01.htm"><font size="-1">欲張りなイヌ</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの江戸小話</font></b><br />
<br />
<a href="../../../../douwa/pc/kobanashi/01/01.htm"><font size="-1">ぞうきんとお年玉</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの百物語</font></b><br />
<br />
<a href="../../../../douwa/pc/kaidan/01/01.htm"><font size="-1">百物語の幽霊</font></a>
</td>
</tr>
</tbody>
</table>
<table width="100%" border="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td height="30" align="center" bgcolor="#C8FFC8">
<b><font size="-1">福娘のサイト</font></b>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>366日への旅</b><br />
<br />
<a href="http://hukumusume.com/366/">毎日の記念日・誕生花 ・有名人の誕生日と性格判断</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>福娘童話集</b><br />
<br />
<a href="http://hukumusume.com/douwa/">世界と日本の童話と昔話</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>女の子応援サイト -さくら-</b><br />
<br />
<a href="http://hukumusume.com/sakura/index.html">誕生日占い、お仕事紹介、おまじない、など</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>子どもの病気相談所</b><br />
<br />
<a href="http://hukumusume.com/my_baby/sick/">病気検索と対応方法、症状から検索するWEB問診</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>世界60秒巡り</b><br />
<br />
<a href="http://hukumusume.com/366/world/">国旗国歌や世界遺産など、世界の国々の豆知識</a></font>
</td>
</tr>
</tbody>
</table>
<td></td>
<td align="right" valign="top">
<img src="../../../../366/logo_bana/corner_2.gif" width="7" height="7"/>
</td>
</tr>
</tbody>
</table>
</body>
</tbody>
</table>
<table width="166" border="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td width="156" height="50">
&#160;&#160;&#160;&#160; <font size="-1"><b>1月 1日の豆知識</b></font><br/>
<br/>
<font size="-2"><u><br/>
<br/>
366日への旅</u></font>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97"
width="1" height="1"/><b><font size="-1">きょうの記念日</font></b><br/>
<br/>
<a href="../../../../366/kinenbi/pc/01gatu/1_01.htm"><font
size="-1">元旦</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/><b><font size="-1">きょうの誕生花</font></b><br/>
<br/>
<a href="../../../../366/hana/pc/01gatu/1_01.htm"><font
size="-1">松(まつ)</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの誕生日・出来事</font></b><br/>
<br/>
<a href="../../../../366/birthday/pc/01gatu/1_01.htm"><font size="-1">1949年　Mr.マリック(マジシャン)</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">恋の誕生日占い</font></b><br/>
<br/>
<a href="../../../../sakura/uranai/birthday/01/01.html"><font size="-1">自分の考えをしっかりと持った女の子。</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">なぞなぞ小学校</font></b><br/>
<br/>
<a href="../../../../nazonazo/new/2012/04/02.html"><font size="-1">○(丸)を取ったらお母さんになってしまう男の人は？</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">あこがれの職業紹介</font></b><br/>
<br/>
<a href="../../../../sakura/navi/work/2017/041.html"><font
size="-1">歌手</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">恋の魔法とおまじない</font></b> 001<br/>
<br/>
<a href="../../../../omajinai/new/2012/00/re01.html"><font size="-1">両思いになれる
おまじない</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#C8FFC8">
<font size="-1">　　<b>1月 1日の童話・昔話</b><br/>
<br/>
<u><font size="-2"><br/>
<br/>
福娘童話集</font></u></font>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの日本昔話</font></b><br/>
<br/>
<a href="../../../../douwa/pc/jap/01/01.htm"><font
size="-1">ネコがネズミを追いかける訳</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの世界昔話<img
src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/></font></b><br/>
<br/>
<a href="../../../../douwa/pc/world/01/01a.htm"><font
size="-1">モンゴルの十二支話</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<img src="file:///C|/Documents%20and%20Settings/%E7%A6%8F%E5%A8%98note/%E3%83%87%E3%82%B9%E3%82%AF%E3%83%88%E3%83%83%E3%83%97/company_website15/image/spacer.gif"
width="1" height="1"/><b><font size="-1">きょうの日本民話</font></b><br/>
<br/>
<a href="../../../../douwa/pc/minwa/01/01c.html"><font
size="-1">仕事の取替えっこ</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうのイソップ童話</font></b><br/>
<br/>
<a href="../../../../douwa/pc/aesop/01/01.htm"><font size="-1">欲張りなイヌ</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの江戸小話</font></b><br/>
<br/>
<a href="../../../../douwa/pc/kobanashi/01/01.htm"><font
size="-1">ぞうきんとお年玉</font></a>
</td>
</tr>
<tr>
<td height="50" bgcolor="#ECFFEC">
<b><font size="-1">きょうの百物語</font></b><br/>
<br/>
<a href="../../../../douwa/pc/kaidan/01/01.htm"><font
size="-1">百物語の幽霊</font></a>
</td>
</tr>
</tbody>
</table>
<table width="100%" border="0" bgcolor="#C8FFC8">
<tbody>
<tr>
<td height="30" align="center" bgcolor="#C8FFC8">
<b><font size="-1">福娘のサイト</font></b>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>366日への旅</b><br/>
<br/>
<a href="http://hukumusume.com/366/">毎日の記念日・誕生花 ・有名人の誕生日と性格判断</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>福娘童話集</b><br/>
<br/>
<a href="http://hukumusume.com/douwa/">世界と日本の童話と昔話</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>女の子応援サイト -さくら-</b><br/>
<br/>
<a href="http://hukumusume.com/sakura/index.html">誕生日占い、お仕事紹介、おまじない、など</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>子どもの病気相談所</b><br/>
<br/>
<a href="http://hukumusume.com/my_baby/sick/">病気検索と対応方法、症状から検索するWEB問診</a></font>
</td>
</tr>
<tr>
<td height="60" bgcolor="#ECFFEC">
<font size="-1"><b>世界60秒巡り</b><br/>
<br/>
<a href="http://hukumusume.com/366/world/">国旗国歌や世界遺産など、世界の国々の豆知識</a></font>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</body>
</html>

99
app/src/main/assets/readability/test/test-pages/iab-1/expected.html
View File

@ -1,31 +1,96 @@
<div id="readability-page-1" class="page">
<div>
<p>We messed up. As technologists, tasked with delivering content and services to users, we lost track of the user experience.</p>
<p>Twenty years ago we saw an explosion of websites, built by developers around the world, providing all forms of content. This was the beginning of an age of enlightenment, the intersection of content and technology. Many of us in the technical field felt compelled, and even empowered, to produce information as the distribution means for mass communication were no longer restricted by a high barrier to entry.</p>
<p>In 2000, the dark ages came when the dot-com bubble burst. We were told that our startups were gone or that our divisions sustained by corporate parent companies needed to be in the black. It was a wakeup call that led to a renaissance age. Digital advertising became the foundation of an economic engine that, still now, sustains the free and democratic World Wide Web. In digital publishing, we strived to balance content, commerce, and technology. The content management systems and communication gateways we built to inform and entertain populations around the world disrupted markets and in some cases governments, informed communities of imminent danger, and liberated new forms of art and entertainment—all while creating a digital middle class of small businesses.</p>
<p>We engineered not just the technical, but also the social and economic foundation that users around the world came to lean on for access to real time information. And users came to expect this information whenever and wherever they needed it. And more often than not, for anybody with a connected device, it was free.</p>
<p>We messed up. As technologists, tasked with delivering content and services to users, we
lost track of the user experience.</p>
<p>Twenty years ago we saw an explosion of websites, built by developers around the world,
providing all forms of content. This was the beginning of an age of enlightenment, the
intersection of content and technology. Many of us in the technical field felt
compelled, and even empowered, to produce information as the distribution means for mass
communication were no longer restricted by a high barrier to entry.</p>
<p>In 2000, the dark ages came when the dot-com bubble burst. We were told that our startups
were gone or that our divisions sustained by corporate parent companies needed to be in
the black. It was a wakeup call that led to a renaissance age. Digital advertising
became the foundation of an economic engine that, still now, sustains the free and
democratic World Wide Web. In digital publishing, we strived to balance content,
commerce, and technology. The content management systems and communication gateways we
built to inform and entertain populations around the world disrupted markets and in some
cases governments, informed communities of imminent danger, and liberated new forms of
art and entertainment—all while creating a digital middle class of small businesses.</p>
<p>We engineered not just the technical, but also the social and economic foundation that
users around the world came to lean on for access to real time information. And users
came to expect this information whenever and wherever they needed it. And more often
than not, for anybody with a connected device, it was free.</p>
<p>This was choice—powered by digital advertising—and premised on user experience.</p>
<p>But we messed up.</p>
<p>Through our pursuit of further automation and maximization of margins during the industrial age of media technology, we built advertising technology to optimize publishers’ yield of marketing budgets that had eroded after the last recession. Looking back now, our scraping of dimes may have cost us dollars in consumer loyalty. The fast, scalable systems of targeting users with ever-heftier advertisements have slowed down the public internet and drained more than a few batteries. We were so clever and so good at it that we over-engineered the capabilities of the plumbing laid down by, well, ourselves. This steamrolled the users, depleted their devices, and tried their patience.</p>
<p>The rise of ad blocking poses a threat to the internet and could potentially drive users to an enclosed platform world dominated by a few companies. We have let the fine equilibrium of content, commerce, and technology get out of balance in the open web. We had, and still do have, a responsibility to educate the business side, and in some cases to push back. We lost sight of our social and ethical responsibility to provide a safe, usable experience for anyone and everyone wanting to consume the content of their choice.</p>
<p>Through our pursuit of further automation and maximization of margins during the
industrial age of media technology, we built advertising technology to optimize
publishers’ yield of marketing budgets that had eroded after the last recession. Looking
back now, our scraping of dimes may have cost us dollars in consumer loyalty. The fast,
scalable systems of targeting users with ever-heftier advertisements have slowed down
the public internet and drained more than a few batteries. We were so clever and so good
at it that we over-engineered the capabilities of the plumbing laid down by, well,
ourselves. This steamrolled the users, depleted their devices, and tried their
patience.</p>
<p>The rise of ad blocking poses a threat to the internet and could potentially drive users
to an enclosed platform world dominated by a few companies. We have let the fine
equilibrium of content, commerce, and technology get out of balance in the open web. We
had, and still do have, a responsibility to educate the business side, and in some cases
to push back. We lost sight of our social and ethical responsibility to provide a safe,
usable experience for anyone and everyone wanting to consume the content of their
choice.</p>
<p>We need to bring that back into alignment, starting right now.</p>
<p><a href="http://www.iab.com/wp-content/uploads/2015/10/getting-lean-with-digital-ad-ux.jpg"><img width="300" height="250" alt="Getting LEAN with Digital Ad UX" src="http://www.iab.com/wp-content/uploads/2015/10/getting-lean-with-digital-ad-ux-300x250.jpg" /></a>Today, the IAB Tech Lab is launching the L.E.A.N. Ads program. Supported by the Executive Committee of the IAB Tech Lab Board, IABs around the world, and hundreds of member companies, L.E.A.N. stands for Light, Encrypted, Ad choice supported, Non-invasive ads. These are principles that will help guide the next phases of advertising technical standards for the global digital advertising supply chain.</p>
<p>As with any other industry, standards should be created by non-profit standards-setting bodies, with many diverse voices providing input. We will invite all parties for public comment, and make sure consumer interest groups have the opportunity to provide input.</p>
<p>L.E.A.N. Ads do not replace the current advertising standards many consumers still enjoy and engage with while consuming content on our sites across all IP enabled devices. Rather, these principles will guide an alternative set of standards that provide choice for marketers, content providers, and consumers.</p>
<p>Among the many areas of concentration, we must also address frequency capping on retargeting in Ad Tech and make sure a user is targeted appropriately before, but never AFTER they make a purchase. If we are so good at reach and scale, we can be just as good, if not better, at moderation. Additionally, we must address volume of ads per page as well as continue on the path to viewability. The dependencies here are critical to an optimized user experience.</p>
<p>The consumer is demanding these actions, challenging us to do better, and we must respond.</p>
<p>The IAB Tech Lab will continue to provide the tools for publishers in the digital supply chain to have a dialogue with users about their choices so that content providers can generate revenue while creating value. Publishers should have the opportunity to provide rich advertising experiences, L.E.A.N. advertising experiences, and subscription services. Or publishers can simply deny their service to users who choose to keep on blocking ads. That is all part of elasticity of consumer tolerance and choice.</p>
<p>Finally, we must do this in an increasingly fragmented market, across screens. We must do this in environments where entire sites are blocked, purposefully or not. Yes, it is disappointing that our development efforts will have to manage with multiple frameworks while we work to supply the economic engine to sustain an open internet. However, our goal is still to provide diverse content and voices to as many connected users as possible around the world.</p>
<p>
<a href="http://www.iab.com/wp-content/uploads/2015/10/getting-lean-with-digital-ad-ux.jpg"><img
width="300" height="250" alt="Getting LEAN with Digital Ad UX"
src="http://www.iab.com/wp-content/uploads/2015/10/getting-lean-with-digital-ad-ux-300x250.jpg"/></a>Today,
the IAB Tech Lab is launching the L.E.A.N. Ads program. Supported by the Executive
Committee of the IAB Tech Lab Board, IABs around the world, and hundreds of member
companies, L.E.A.N. stands for Light, Encrypted, Ad choice supported, Non-invasive ads.
These are principles that will help guide the next phases of advertising technical
standards for the global digital advertising supply chain.</p>
<p>As with any other industry, standards should be created by non-profit standards-setting
bodies, with many diverse voices providing input. We will invite all parties for public
comment, and make sure consumer interest groups have the opportunity to provide
input.</p>
<p>L.E.A.N. Ads do not replace the current advertising standards many consumers still enjoy
and engage with while consuming content on our sites across all IP enabled devices.
Rather, these principles will guide an alternative set of standards that provide choice
for marketers, content providers, and consumers.</p>
<p>Among the many areas of concentration, we must also address frequency capping on
retargeting in Ad Tech and make sure a user is targeted appropriately before, but never
AFTER they make a purchase. If we are so good at reach and scale, we can be just as
good, if not better, at moderation. Additionally, we must address volume of ads per page
as well as continue on the path to viewability. The dependencies here are critical to an
optimized user experience.</p>
<p>The consumer is demanding these actions, challenging us to do better, and we must
respond.</p>
<p>The IAB Tech Lab will continue to provide the tools for publishers in the digital supply
chain to have a dialogue with users about their choices so that content providers can
generate revenue while creating value. Publishers should have the opportunity to provide
rich advertising experiences, L.E.A.N. advertising experiences, and subscription
services. Or publishers can simply deny their service to users who choose to keep on
blocking ads. That is all part of elasticity of consumer tolerance and choice.</p>
<p>Finally, we must do this in an increasingly fragmented market, across screens. We must do
this in environments where entire sites are blocked, purposefully or not. Yes, it is
disappointing that our development efforts will have to manage with multiple frameworks
while we work to supply the economic engine to sustain an open internet. However, our
goal is still to provide diverse content and voices to as many connected users as
possible around the world.</p>
<p>That is user experience.</p>
<p>IAB Tech Lab Members can join the IAB Tech Lab Ad Blocking Working Group, please email <a href="mailto:adblocking@iab.com">adblocking@iab.com</a> for more information.</p>
<p>Read <a target="_blank" href="http://www.iab.com/insights/ad-blocking/">more about ad blocking here</a>.</p>
<p>IAB Tech Lab Members can join the IAB Tech Lab Ad Blocking Working Group, please email <a
href="mailto:adblocking@iab.com">adblocking@iab.com</a> for more information.</p>
<p>Read <a target="_blank" href="http://www.iab.com/insights/ad-blocking/">more about ad
blocking here</a>.</p>
</div>
<div id="post-author">
<figure><img alt="Auto Draft 14" src="http://www.iab.com/wp-content/uploads/2015/05/auto-draft-16-150x150.jpg" /></figure>
<figure><img alt="Auto Draft 14"
src="http://www.iab.com/wp-content/uploads/2015/05/auto-draft-16-150x150.jpg"/>
</figure>
<div>
<h4>About the author</h4>
<p><strong>Scott Cunningham</strong></p>
<p>Senior Vice President of Technology and Ad Operations at IAB, and General Manager of the IAB Tech Lab</p>
<p>Senior Vice President of Technology and Ad Operations at IAB, and General Manager of
the IAB Tech Lab</p>
</div>
</div>
</div>

3251
app/src/main/assets/readability/test/test-pages/iab-1/source.html
View File

File diff suppressed because one or more lines are too long

147
app/src/main/assets/readability/test/test-pages/ietf-1/expected.html
View File

@ -1,8 +1,26 @@
<div id="readability-page-1" class="page">
<span>[<a href="http://fakehost/html/" title="Document search and retrieval page">Docs</a>] [<a href="https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt" title="Plaintext version of this document">txt</a>|<a href="http://fakehost/pdf/draft-dejong-remotestorage-04.txt" title="PDF version of this document">pdf</a>] [<a href="https://datatracker.ietf.org/doc/draft-dejong-remotestorage" title="IESG Datatracker information for this document">Tracker</a>] [<a href="mailto:draft-dejong-remotestorage@tools.ietf.org?subject=draft-dejong-remotestorage%20" title="Send email to the document authors">Email</a>] [<a href="http://fakehost/rfcdiff?difftype=--hwdiff&amp;url2=draft-dejong-remotestorage-04.txt" title="Inline diff (wdiff)">Diff1</a>] [<a href="http://fakehost/rfcdiff?url2=draft-dejong-remotestorage-04.txt" title="Side-by-side diff">Diff2</a>] [<a href="http://fakehost/idnits?url=https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt" title="Run an idnits check of this document">Nits</a>] </span><br />
<span> </span><br />
<span>Versions: <a href="http://fakehost/test/draft-dejong-remotestorage-00">00</a> <a href="http://fakehost/test/draft-dejong-remotestorage-01">01</a> <a href="http://fakehost/test/draft-dejong-remotestorage-02">02</a> <a href="http://fakehost/test/draft-dejong-remotestorage-03">03</a> <a href="http://fakehost/test/draft-dejong-remotestorage-04">04</a> </span><br />
<span> </span><br />
<span>[<a href="http://fakehost/html/" title="Document search and retrieval page">Docs</a>] [<a
href="https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt"
title="Plaintext version of this document">txt</a>|<a
href="http://fakehost/pdf/draft-dejong-remotestorage-04.txt"
title="PDF version of this document">pdf</a>] [<a
href="https://datatracker.ietf.org/doc/draft-dejong-remotestorage"
title="IESG Datatracker information for this document">Tracker</a>] [<a
href="mailto:draft-dejong-remotestorage@tools.ietf.org?subject=draft-dejong-remotestorage%20"
title="Send email to the document authors">Email</a>] [<a
href="http://fakehost/rfcdiff?difftype=--hwdiff&amp;url2=draft-dejong-remotestorage-04.txt"
title="Inline diff (wdiff)">Diff1</a>] [<a
href="http://fakehost/rfcdiff?url2=draft-dejong-remotestorage-04.txt"
title="Side-by-side diff">Diff2</a>] [<a
href="http://fakehost/idnits?url=https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt"
title="Run an idnits check of this document">Nits</a>] </span><br/>
<span> </span><br/>
<span>Versions: <a href="http://fakehost/test/draft-dejong-remotestorage-00">00</a> <a
href="http://fakehost/test/draft-dejong-remotestorage-01">01</a> <a
href="http://fakehost/test/draft-dejong-remotestorage-02">02</a> <a
href="http://fakehost/test/draft-dejong-remotestorage-03">03</a> <a
href="http://fakehost/test/draft-dejong-remotestorage-04">04</a> </span><br/>
<span> </span><br/>
<pre>INTERNET DRAFT Michiel B. de Jong
Document: <a href="http://fakehost/test/draft-dejong-remotestorage-04">draft-dejong-remotestorage-04</a> IndieHosters
F. Kooman
@ -25,7 +43,8 @@ Abstract
Status of this Memo
This Internet-Draft is submitted in full conformance with the
provisions of <a href="http://fakehost/test/bcp78">BCP 78</a> and <a href="http://fakehost/test/bcp79">BCP 79</a>.
provisions of <a href="http://fakehost/test/bcp78">BCP 78</a> and <a
href="http://fakehost/test/bcp79">BCP 79</a>.
Internet-Drafts are working documents of the Internet Engineering
Task Force (IETF). Note that other groups may also distribute
@ -63,34 +82,62 @@ Copyright Notice
Table of Contents
<a href="#section-1">1</a>. Introduction...................................................<a href="#page-2">2</a>
<a href="#section-2">2</a>. Terminology....................................................<a href="#page-3">3</a>
<a href="#section-3">3</a>. Storage model..................................................<a href="#page-3">3</a>
<a href="#section-4">4</a>. Requests.......................................................<a href="#page-4">4</a>
<a href="#section-5">5</a>. Response codes.................................................<a href="#page-7">7</a>
<a href="#section-6">6</a>. Versioning.....................................................<a href="#page-7">7</a>
<a href="#section-7">7</a>. CORS headers...................................................<a href="#page-8">8</a>
<a href="#section-8">8</a>. Session description............................................<a href="#page-8">8</a>
<a href="#section-9">9</a>. Bearer tokens and access control...............................<a href="#page-9">9</a>
<a href="#section-10">10</a>. Application-first bearer token issuance.......................<a href="#page-10">10</a>
<a href="#section-11">11</a>. Storage-first bearer token issuance...........................<a href="#page-11">11</a>
<a href="#section-12">12</a>. Example wire transcripts......................................<a href="#page-12">12</a>
<a href="#section-12.1">12.1</a>. WebFinger................................................<a href="#page-12">12</a>
<a href="#section-12.2">12.2</a>. OAuth dialog form........................................<a href="#page-13">13</a>
<a href="#section-12.3">12.3</a>. OAuth dialog form submission.............................<a href="#page-14">14</a>
<a href="#section-12.4">12.4</a>. OPTIONS preflight........................................<a href="#page-15">15</a>
<a href="#section-12.5">12.5</a>. Initial PUT..............................................<a href="#page-15">15</a>
<a href="#section-12.6">12.6</a>. Subsequent PUT...........................................<a href="#page-16">16</a>
<a href="#section-12.7">12.7</a>. GET......................................................<a href="#page-16">16</a>
<a href="#section-12.8">12.8</a>. DELETE...................................................<a href="#page-17">17</a>
<a href="#section-13">13</a>. Distributed versioning........................................<a href="#page-17">17</a>
<a href="#section-14">14</a>. Security Considerations.......................................<a href="#page-19">19</a>
<a href="#section-15">15</a>. IANA Considerations...........................................<a href="#page-20">20</a>
<a href="#section-16">16</a>. Acknowledgments...............................................<a href="#page-20">20</a>
<a href="#section-17">17</a>. References....................................................<a href="#page-21">21</a>
<a href="#section-17.1">17.1</a>. Normative References.....................................<a href="#page-21">21</a>
<a href="#section-17.2">17.2</a>. Informative References...................................<a href="#page-21">21</a>
<a href="#section-18">18</a>. Authors' addresses............................................<a href="#page-22">22</a>
<a href="#section-1">1</a>. Introduction...................................................<a
href="#page-2">2</a>
<a href="#section-2">2</a>. Terminology....................................................<a
href="#page-3">3</a>
<a href="#section-3">3</a>. Storage model..................................................<a
href="#page-3">3</a>
<a href="#section-4">4</a>. Requests.......................................................<a
href="#page-4">4</a>
<a href="#section-5">5</a>. Response codes.................................................<a
href="#page-7">7</a>
<a href="#section-6">6</a>. Versioning.....................................................<a
href="#page-7">7</a>
<a href="#section-7">7</a>. CORS headers...................................................<a
href="#page-8">8</a>
<a href="#section-8">8</a>. Session description............................................<a
href="#page-8">8</a>
<a href="#section-9">9</a>. Bearer tokens and access control...............................<a
href="#page-9">9</a>
<a href="#section-10">10</a>. Application-first bearer token issuance.......................<a
href="#page-10">10</a>
<a href="#section-11">11</a>. Storage-first bearer token issuance...........................<a
href="#page-11">11</a>
<a href="#section-12">12</a>. Example wire transcripts......................................<a
href="#page-12">12</a>
<a href="#section-12.1">12.1</a>. WebFinger................................................<a
href="#page-12">12</a>
<a href="#section-12.2">12.2</a>. OAuth dialog form........................................<a
href="#page-13">13</a>
<a href="#section-12.3">12.3</a>. OAuth dialog form submission.............................<a
href="#page-14">14</a>
<a href="#section-12.4">12.4</a>. OPTIONS preflight........................................<a
href="#page-15">15</a>
<a href="#section-12.5">12.5</a>. Initial PUT..............................................<a
href="#page-15">15</a>
<a href="#section-12.6">12.6</a>. Subsequent PUT...........................................<a
href="#page-16">16</a>
<a href="#section-12.7">12.7</a>. GET......................................................<a
href="#page-16">16</a>
<a href="#section-12.8">12.8</a>. DELETE...................................................<a
href="#page-17">17</a>
<a href="#section-13">13</a>. Distributed versioning........................................<a
href="#page-17">17</a>
<a href="#section-14">14</a>. Security Considerations.......................................<a
href="#page-19">19</a>
<a href="#section-15">15</a>. IANA Considerations...........................................<a
href="#page-20">20</a>
<a href="#section-16">16</a>. Acknowledgments...............................................<a
href="#page-20">20</a>
<a href="#section-17">17</a>. References....................................................<a
href="#page-21">21</a>
<a href="#section-17.1">17.1</a>. Normative References.....................................<a
href="#page-21">21</a>
<a href="#section-17.2">17.2</a>. Informative References...................................<a
href="#page-21">21</a>
<a href="#section-18">18</a>. Authors' addresses............................................<a
href="#page-22">22</a>
<span><a name="section-1" href="#section-1">1</a>. Introduction</span>
@ -131,7 +178,8 @@ Table of Contents
The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
document are to be interpreted as described in <a href="http://fakehost/test/rfc2119">RFC 2119</a> [<a href="#ref-WORDS">WORDS</a>].
document are to be interpreted as described in <a
href="http://fakehost/test/rfc2119">RFC 2119</a> [<a href="#ref-WORDS">WORDS</a>].
"SHOULD" and "SHOULD NOT" are appropriate when valid exceptions to a
general requirement are known to exist or appear to exist, and it is
@ -641,13 +689,16 @@ g.com HTTP/1.1
"href": "<a href="https://michielbdejong.com:7678/inbox">https://michielbdejong.com:7678/inbox</a>",
"rel": "post-me-anything"
}, {
"href": "<a href="https://michielbdejong.com/me.jpg">https://michielbdejong.com/me.jpg</a>",
"href": "<a
href="https://michielbdejong.com/me.jpg">https://michielbdejong.com/me.jpg</a>",
"rel": "avatar"
}, {
"href": "<a href="https://3pp.io:4439/storage/michiel">https://3pp.io:4439/storage/michiel</a>",
"rel": "remotestorage",
"properties": {
"<a href="http://remotestorage.io/spec/version">http://remotestorage.io/spec/version</a>": "<a href="http://fakehost/test/draft-dejong-re">draft-dejong-re</a>\
"<a
href="http://remotestorage.io/spec/version">http://remotestorage.io/spec/version</a>": "<a
href="http://fakehost/test/draft-dejong-re">draft-dejong-re</a>\
motestorage-04",
"<a href="http://tools.ietf.org/html/rfc6749#section-4.2">http://tools.ietf.org/html/rfc6749#section-4.2</a>": "https\
://3pp.io:4439/oauth/michiel",
@ -753,7 +804,8 @@ ntent-Type, Origin, X-Requested-With, If-Match, If-None-Match
Origin: <a href="https://drinks-unhosted.5apps.com/">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Content-Type: application/json; charset=UTF-8
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-None-Match: *
{"name":"test","@context":"<a href="http://remotestorage/">http://remotestorage</a>.io/spec/modules\
@ -785,7 +837,8 @@ ntent-Type, Origin, X-Requested-With, If-Match, If-None-Match
Origin: <a href="https://drinks-unhosted.5apps.com/">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Content-Type: application/json; charset=UTF-8
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-Match: "1382694045000"
{"name":"test", "updated":true, "@context":"http://remotestorag\
@ -806,7 +859,8 @@ e.io/spec/modules/myfavoritedrinks/drink"}
Host: 3pp.io:4439
Origin: <a href="https://drinks-unhosted.5apps.com/">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-None-Match: "1382694045000", "1382694048000"
And the server may respond with a 304 Not Modified status:
@ -866,7 +920,8 @@ charset=UTF-8","Content-Length":106}}}
Origin: <a href="https://drinks-unhosted.5apps.com/">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Content-Type: application/json; charset=UTF-8
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-Match: "1382694045000"
@ -1021,7 +1076,8 @@ charset=UTF-8","Content-Length":106}}}
[<a name="ref-WORDS" id="ref-WORDS">WORDS</a>]
Bradner, S., "Key words for use in RFCs to Indicate Requirement
Levels", <a href="http://fakehost/test/bcp14">BCP 14</a>, <a href="http://fakehost/test/rfc2119">RFC 2119</a>, March 1997.
Levels", <a href="http://fakehost/test/bcp14">BCP 14</a>, <a
href="http://fakehost/test/rfc2119">RFC 2119</a>, March 1997.
<span>de Jong [Page 20]</span>
@ -1061,7 +1117,8 @@ charset=UTF-8","Content-Length":106}}}
Conditional Requests", <a href="http://fakehost/test/rfc7233">RFC7233</a>, June 2014.
[<a name="ref-SPDY" id="ref-SPDY">SPDY</a>]
Mark Belshe, Roberto Peon, "SPDY Protocol - Draft 3.1", <a href="http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1">http://</a>
Mark Belshe, Roberto Peon, "SPDY Protocol - Draft 3.1", <a
href="http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1">http://</a>
<a href="http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1">www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1</a>,
September 2013.
@ -1128,7 +1185,9 @@ charset=UTF-8","Content-Length":106}}}
de Jong [Page 22]
</pre><br />
<span><small><small>Html markup produced by rfcmarkup 1.111, available from <a href="https://tools.ietf.org/tools/rfcmarkup/">https://tools.ietf.org/tools/rfcmarkup/</a>
</pre>
<br/>
<span><small><small>Html markup produced by rfcmarkup 1.111, available from <a
href="https://tools.ietf.org/tools/rfcmarkup/">https://tools.ietf.org/tools/rfcmarkup/</a>
</small></small></span>
</div>

268
app/src/main/assets/readability/test/test-pages/ietf-1/source.html
View File

@ -1,26 +1,26 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head profile="http://dublincore.org/documents/2008/08/04/dc-html/">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="robots" content="index,follow" />
<meta name="creator" content="rfcmarkup version 1.111" />
<link rel="schema.DC" href="http://purl.org/dc/elements/1.1/" />
<meta name="DC.Identifier" content="urn:ietf:id:dejong-remotestorage" />
<meta name="DC.Description.Abstract" content="This draft describes a protocol by which client-side applications,
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta name="robots" content="index,follow"/>
<meta name="creator" content="rfcmarkup version 1.111"/>
<link rel="schema.DC" href="http://purl.org/dc/elements/1.1/"/>
<meta name="DC.Identifier" content="urn:ietf:id:dejong-remotestorage"/>
<meta name="DC.Description.Abstract" content="This draft describes a protocol by which client-side applications,
running inside a web browser, can communicate with a data storage
server that is hosted on a different domain name. This way, the
provider of a web application need not also play the role of data
storage provider. The protocol supports storing, retrieving, and
removing individual documents, as well as listing the contents of an
individual folder, and access control is based on bearer tokens." />
<meta name="DC.Creator" content="Kooman, F." />
<meta name="DC.Creator" content="Jong, Michiel de" />
<meta name="DC.Date.Issued" content="2014-12-15" />
<meta name="DC.Title" content="remoteStorage" />
individual folder, and access control is based on bearer tokens."/>
<meta name="DC.Creator" content="Kooman, F."/>
<meta name="DC.Creator" content="Jong, Michiel de"/>
<meta name="DC.Date.Issued" content="2014-12-15"/>
<meta name="DC.Title" content="remoteStorage"/>
<link rel="icon" href="/images/id.png" type="image/png" />
<link rel="shortcut icon" href="/images/id.png" type="image/png" />
<link rel="icon" href="/images/id.png" type="image/png"/>
<link rel="shortcut icon" href="/images/id.png" type="image/png"/>
<title>draft-dejong-remotestorage-04 - remoteStorage</title>
@ -102,6 +102,7 @@ individual folder, and access control is based on bearer tokens." />
.legend { font-size: 90%; }
.cplate { font-size: 70%; border: solid grey 1px; }
}
</style>
<!--[if IE]>
<style>
@ -109,6 +110,7 @@ individual folder, and access control is based on bearer tokens." />
font-size: 13px;
margin: 10px 10px;
}
</style>
<![endif]-->
@ -137,27 +139,45 @@ individual folder, and access control is based on bearer tokens." />
elem.innerHTML = "";
}
// -->
</script>
</head>
<body onload="addHeaderTags()">
<div style="height: 13px;">
<div onmouseover="this.style.cursor='pointer';"
<div style="height: 13px;">
<div onmouseover="this.style.cursor='pointer';"
onclick="showElem('legend');"
onmouseout="hideElem('legend')"
style="height: 6px; position: absolute;"
style="height: 6px; position: absolute;"
class="pre noprint docinfo bgred"
title="Click for colour legend." > </div>
<div id="legend"
class="docinfo noprint pre legend"
style="position:absolute; top: 4px; left: 4ex; visibility:hidden; background-color: white; padding: 4px 9px 5px 7px; border: solid #345 1px; "
onmouseover="showElem('legend');"
onmouseout="hideElem('legend');">
</div>
</div>
<span class="pre noprint docinfo top">[<a href="../html/" title="Document search and retrieval page">Docs</a>] [<a href="https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt" title="Plaintext version of this document">txt</a>|<a href="/pdf/draft-dejong-remotestorage-04.txt" title="PDF version of this document">pdf</a>] [<a href='https://datatracker.ietf.org/doc/draft-dejong-remotestorage' title='IESG Datatracker information for this document'>Tracker</a>] [<a href="mailto:draft-dejong-remotestorage@tools.ietf.org?subject=draft-dejong-remotestorage%20" title="Send email to the document authors">Email</a>] [<a href="/rfcdiff?difftype=--hwdiff&amp;url2=draft-dejong-remotestorage-04.txt" title="Inline diff (wdiff)">Diff1</a>] [<a href="/rfcdiff?url2=draft-dejong-remotestorage-04.txt" title="Side-by-side diff">Diff2</a>] [<a href="/idnits?url=https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt" title="Run an idnits check of this document">Nits</a>] </span><br />
<span class="pre noprint docinfo"> </span><br />
<span class="pre noprint docinfo">Versions: <a href="./draft-dejong-remotestorage-00">00</a> <a href="./draft-dejong-remotestorage-01">01</a> <a href="./draft-dejong-remotestorage-02">02</a> <a href="./draft-dejong-remotestorage-03">03</a> <a href="./draft-dejong-remotestorage-04">04</a> </span><br />
<span class="pre noprint docinfo"> </span><br />
title="Click for colour legend."></div>
<div id="legend"
class="docinfo noprint pre legend"
style="position:absolute; top: 4px; left: 4ex; visibility:hidden; background-color: white; padding: 4px 9px 5px 7px; border: solid #345 1px; "
onmouseover="showElem('legend');"
onmouseout="hideElem('legend');">
</div>
</div>
<span class="pre noprint docinfo top">[<a href="../html/"
title="Document search and retrieval page">Docs</a>] [<a
href="https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt"
title="Plaintext version of this document">txt</a>|<a
href="/pdf/draft-dejong-remotestorage-04.txt" title="PDF version of this document">pdf</a>] [<a
href='https://datatracker.ietf.org/doc/draft-dejong-remotestorage'
title='IESG Datatracker information for this document'>Tracker</a>] [<a
href="mailto:draft-dejong-remotestorage@tools.ietf.org?subject=draft-dejong-remotestorage%20"
title="Send email to the document authors">Email</a>] [<a
href="/rfcdiff?difftype=--hwdiff&amp;url2=draft-dejong-remotestorage-04.txt"
title="Inline diff (wdiff)">Diff1</a>] [<a
href="/rfcdiff?url2=draft-dejong-remotestorage-04.txt" title="Side-by-side diff">Diff2</a>] [<a
href="/idnits?url=https://tools.ietf.org/id/draft-dejong-remotestorage-04.txt"
title="Run an idnits check of this document">Nits</a>] </span><br/>
<span class="pre noprint docinfo"> </span><br/>
<span class="pre noprint docinfo">Versions: <a href="./draft-dejong-remotestorage-00">00</a> <a
href="./draft-dejong-remotestorage-01">01</a> <a
href="./draft-dejong-remotestorage-02">02</a> <a
href="./draft-dejong-remotestorage-03">03</a> <a
href="./draft-dejong-remotestorage-04">04</a> </span><br/>
<span class="pre noprint docinfo"> </span><br/>
<pre>INTERNET DRAFT Michiel B. de Jong
Document: <a href="./draft-dejong-remotestorage-04">draft-dejong-remotestorage-04</a> IndieHosters
F. Kooman
@ -211,40 +231,69 @@ Copyright Notice
<span class="grey">de Jong [Page 1]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-2" id="page-2" href="#page-2" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-2" id="page-2" href="#page-2" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
Table of Contents
<a href="#section-1">1</a>. Introduction...................................................<a href="#page-2">2</a>
<a href="#section-2">2</a>. Terminology....................................................<a href="#page-3">3</a>
<a href="#section-3">3</a>. Storage model..................................................<a href="#page-3">3</a>
<a href="#section-4">4</a>. Requests.......................................................<a href="#page-4">4</a>
<a href="#section-5">5</a>. Response codes.................................................<a href="#page-7">7</a>
<a href="#section-6">6</a>. Versioning.....................................................<a href="#page-7">7</a>
<a href="#section-7">7</a>. CORS headers...................................................<a href="#page-8">8</a>
<a href="#section-8">8</a>. Session description............................................<a href="#page-8">8</a>
<a href="#section-9">9</a>. Bearer tokens and access control...............................<a href="#page-9">9</a>
<a href="#section-10">10</a>. Application-first bearer token issuance.......................<a href="#page-10">10</a>
<a href="#section-11">11</a>. Storage-first bearer token issuance...........................<a href="#page-11">11</a>
<a href="#section-12">12</a>. Example wire transcripts......................................<a href="#page-12">12</a>
<a href="#section-12.1">12.1</a>. WebFinger................................................<a href="#page-12">12</a>
<a href="#section-12.2">12.2</a>. OAuth dialog form........................................<a href="#page-13">13</a>
<a href="#section-12.3">12.3</a>. OAuth dialog form submission.............................<a href="#page-14">14</a>
<a href="#section-12.4">12.4</a>. OPTIONS preflight........................................<a href="#page-15">15</a>
<a href="#section-12.5">12.5</a>. Initial PUT..............................................<a href="#page-15">15</a>
<a href="#section-12.6">12.6</a>. Subsequent PUT...........................................<a href="#page-16">16</a>
<a href="#section-12.7">12.7</a>. GET......................................................<a href="#page-16">16</a>
<a href="#section-12.8">12.8</a>. DELETE...................................................<a href="#page-17">17</a>
<a href="#section-13">13</a>. Distributed versioning........................................<a href="#page-17">17</a>
<a href="#section-14">14</a>. Security Considerations.......................................<a href="#page-19">19</a>
<a href="#section-15">15</a>. IANA Considerations...........................................<a href="#page-20">20</a>
<a href="#section-16">16</a>. Acknowledgments...............................................<a href="#page-20">20</a>
<a href="#section-17">17</a>. References....................................................<a href="#page-21">21</a>
<a href="#section-17.1">17.1</a>. Normative References.....................................<a href="#page-21">21</a>
<a href="#section-17.2">17.2</a>. Informative References...................................<a href="#page-21">21</a>
<a href="#section-18">18</a>. Authors' addresses............................................<a href="#page-22">22</a>
<a href="#section-1">1</a>. Introduction...................................................<a
href="#page-2">2</a>
<a href="#section-2">2</a>. Terminology....................................................<a
href="#page-3">3</a>
<a href="#section-3">3</a>. Storage model..................................................<a
href="#page-3">3</a>
<a href="#section-4">4</a>. Requests.......................................................<a
href="#page-4">4</a>
<a href="#section-5">5</a>. Response codes.................................................<a
href="#page-7">7</a>
<a href="#section-6">6</a>. Versioning.....................................................<a
href="#page-7">7</a>
<a href="#section-7">7</a>. CORS headers...................................................<a
href="#page-8">8</a>
<a href="#section-8">8</a>. Session description............................................<a
href="#page-8">8</a>
<a href="#section-9">9</a>. Bearer tokens and access control...............................<a
href="#page-9">9</a>
<a href="#section-10">10</a>. Application-first bearer token issuance.......................<a
href="#page-10">10</a>
<a href="#section-11">11</a>. Storage-first bearer token issuance...........................<a
href="#page-11">11</a>
<a href="#section-12">12</a>. Example wire transcripts......................................<a
href="#page-12">12</a>
<a href="#section-12.1">12.1</a>. WebFinger................................................<a
href="#page-12">12</a>
<a href="#section-12.2">12.2</a>. OAuth dialog form........................................<a
href="#page-13">13</a>
<a href="#section-12.3">12.3</a>. OAuth dialog form submission.............................<a
href="#page-14">14</a>
<a href="#section-12.4">12.4</a>. OPTIONS preflight........................................<a
href="#page-15">15</a>
<a href="#section-12.5">12.5</a>. Initial PUT..............................................<a
href="#page-15">15</a>
<a href="#section-12.6">12.6</a>. Subsequent PUT...........................................<a
href="#page-16">16</a>
<a href="#section-12.7">12.7</a>. GET......................................................<a
href="#page-16">16</a>
<a href="#section-12.8">12.8</a>. DELETE...................................................<a
href="#page-17">17</a>
<a href="#section-13">13</a>. Distributed versioning........................................<a
href="#page-17">17</a>
<a href="#section-14">14</a>. Security Considerations.......................................<a
href="#page-19">19</a>
<a href="#section-15">15</a>. IANA Considerations...........................................<a
href="#page-20">20</a>
<a href="#section-16">16</a>. Acknowledgments...............................................<a
href="#page-20">20</a>
<a href="#section-17">17</a>. References....................................................<a
href="#page-21">21</a>
<a href="#section-17.1">17.1</a>. Normative References.....................................<a
href="#page-21">21</a>
<a href="#section-17.2">17.2</a>. Informative References...................................<a
href="#page-21">21</a>
<a href="#section-18">18</a>. Authors' addresses............................................<a
href="#page-22">22</a>
<span class="h2"><a class="selflink" name="section-1" href="#section-1">1</a>. Introduction</span>
@ -261,7 +310,8 @@ Table of Contents
<span class="grey">de Jong [Page 2]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-3" id="page-3" href="#page-3" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-3" id="page-3" href="#page-3" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -284,7 +334,8 @@ Table of Contents
The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
document are to be interpreted as described in <a href="./rfc2119">RFC 2119</a> [<a href="#ref-WORDS">WORDS</a>].
document are to be interpreted as described in <a href="./rfc2119">RFC 2119</a> [<a
href="#ref-WORDS">WORDS</a>].
"SHOULD" and "SHOULD NOT" are appropriate when valid exceptions to a
general requirement are known to exist or appear to exist, and it is
@ -311,7 +362,8 @@ Table of Contents
<span class="grey">de Jong [Page 3]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-4" id="page-4" href="#page-4" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-4" id="page-4" href="#page-4" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -361,7 +413,8 @@ Table of Contents
<span class="grey">de Jong [Page 4]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-5" id="page-5" href="#page-5" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-5" id="page-5" href="#page-5" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -411,7 +464,8 @@ Table of Contents
<span class="grey">de Jong [Page 5]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-6" id="page-6" href="#page-6" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-6" id="page-6" href="#page-6" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -461,7 +515,8 @@ Table of Contents
<span class="grey">de Jong [Page 6]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-7" id="page-7" href="#page-7" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-7" id="page-7" href="#page-7" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -511,7 +566,8 @@ Table of Contents
<span class="grey">de Jong [Page 7]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-8" id="page-8" href="#page-8" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-8" id="page-8" href="#page-8" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -535,7 +591,8 @@ Table of Contents
origin MAY be returned, but for PUT and DELETE requests, the
response MUST echo back the Origin header sent by the client.
<span class="h2"><a class="selflink" name="section-8" href="#section-8">8</a>. Session description</span>
<span class="h2"><a class="selflink" name="section-8"
href="#section-8">8</a>. Session description</span>
The information that a client needs to receive in order to be able
to connect to a server SHOULD reach the client as described in the
@ -561,7 +618,8 @@ Table of Contents
<span class="grey">de Jong [Page 8]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-9" id="page-9" href="#page-9" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-9" id="page-9" href="#page-9" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -611,7 +669,8 @@ Table of Contents
<span class="grey">de Jong [Page 9]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-10" id="page-10" href="#page-10" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-10" id="page-10" href="#page-10" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -661,7 +720,8 @@ Table of Contents
<span class="grey">de Jong [Page 10]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-11" id="page-11" href="#page-11" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-11" id="page-11" href="#page-11" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -711,7 +771,8 @@ Table of Contents
<span class="grey">de Jong [Page 11]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-12" id="page-12" href="#page-12" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-12" id="page-12" href="#page-12" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -754,14 +815,16 @@ Table of Contents
The following examples are not normative ("\" indicates a line was
wrapped).
<span class="h3"><a class="selflink" name="section-12.1" href="#section-12.1">12.1</a>. WebFinger</span>
<span class="h3"><a class="selflink" name="section-12.1"
href="#section-12.1">12.1</a>. WebFinger</span>
In application-first, an in-browser application might issue the
following request, using XMLHttpRequest and CORS:
<span class="grey">de Jong [Page 12]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-13" id="page-13" href="#page-13" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-13" id="page-13" href="#page-13" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -784,13 +847,16 @@ g.com HTTP/1.1
"href": "<a href="https://michielbdejong.com:7678/inbox">https://michielbdejong.com:7678/inbox</a>",
"rel": "post-me-anything"
}, {
"href": "<a href="https://michielbdejong.com/me.jpg">https://michielbdejong.com/me.jpg</a>",
"href": "<a
href="https://michielbdejong.com/me.jpg">https://michielbdejong.com/me.jpg</a>",
"rel": "avatar"
}, {
"href": "<a href="https://3pp.io:4439/storage/michiel">https://3pp.io:4439/storage/michiel</a>",
"rel": "remotestorage",
"properties": {
"<a href="http://remotestorage.io/spec/version">http://remotestorage.io/spec/version</a>": "<a href="./draft-dejong-re">draft-dejong-re</a>\
"<a
href="http://remotestorage.io/spec/version">http://remotestorage.io/spec/version</a>": "<a
href="./draft-dejong-re">draft-dejong-re</a>\
motestorage-04",
"<a href="http://tools.ietf.org/html/rfc6749#section-4.2">http://tools.ietf.org/html/rfc6749#section-4.2</a>": "https\
://3pp.io:4439/oauth/michiel",
@ -811,7 +877,8 @@ motestorage-04",
<span class="grey">de Jong [Page 13]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-14" id="page-14" href="#page-14" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-14" id="page-14" href="#page-14" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -861,7 +928,8 @@ XjzzzHNjkd1CJxoQubA1o%3D&amp;token_type=bearer&amp;state=
<span class="grey">de Jong [Page 14]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-15" id="page-15" href="#page-15" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-15" id="page-15" href="#page-15" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -894,7 +962,8 @@ ntent-Type, Origin, X-Requested-With, If-Match, If-None-Match
Origin: <a href="https://drinks-unhosted.5apps.com">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Content-Type: application/json; charset=UTF-8
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-None-Match: *
{"name":"test","@context":"<a href="http://remotestorage">http://remotestorage</a>.io/spec/modules\
@ -911,7 +980,8 @@ ntent-Type, Origin, X-Requested-With, If-Match, If-None-Match
<span class="grey">de Jong [Page 15]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-16" id="page-16" href="#page-16" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-16" id="page-16" href="#page-16" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -925,7 +995,8 @@ ntent-Type, Origin, X-Requested-With, If-Match, If-None-Match
Origin: <a href="https://drinks-unhosted.5apps.com">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Content-Type: application/json; charset=UTF-8
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-Match: "1382694045000"
{"name":"test", "updated":true, "@context":"http://remotestorag\
@ -946,7 +1017,8 @@ e.io/spec/modules/myfavoritedrinks/drink"}
Host: 3pp.io:4439
Origin: <a href="https://drinks-unhosted.5apps.com">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-None-Match: "1382694045000", "1382694048000"
And the server may respond with a 304 Not Modified status:
@ -961,7 +1033,8 @@ e.io/spec/modules/myfavoritedrinks/drink"}
<span class="grey">de Jong [Page 16]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-17" id="page-17" href="#page-17" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-17" id="page-17" href="#page-17" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -996,7 +1069,8 @@ charset=UTF-8","Content-Length":106}}}
HTTP/1.1 404 Not Found
Access-Control-Allow-Origin: <a href="https://drinks-unhosted.5apps.com">https://drinks-unhosted.5apps.com</a>
<span class="h3"><a class="selflink" name="section-12.8" href="#section-12.8">12.8</a>. DELETE</span>
<span class="h3"><a class="selflink" name="section-12.8"
href="#section-12.8">12.8</a>. DELETE</span>
A DELETE request may look like this:
@ -1005,13 +1079,15 @@ charset=UTF-8","Content-Length":106}}}
Origin: <a href="https://drinks-unhosted.5apps.com">https://drinks-unhosted.5apps.com</a>
Authorization: Bearer j2YnGtXjzzzHNjkd1CJxoQubA1o=
Content-Type: application/json; charset=UTF-8
Referer: <a href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
Referer: <a
href="https://drinks-unhosted.5apps.com/?">https://drinks-unhosted.5apps.com/?</a>
If-Match: "1382694045000"
<span class="grey">de Jong [Page 17]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-18" id="page-18" href="#page-18" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-18" id="page-18" href="#page-18" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -1061,7 +1137,8 @@ charset=UTF-8","Content-Length":106}}}
<span class="grey">de Jong [Page 18]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-19" id="page-19" href="#page-19" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-19" id="page-19" href="#page-19" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -1111,7 +1188,8 @@ charset=UTF-8","Content-Length":106}}}
<span class="grey">de Jong [Page 19]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-20" id="page-20" href="#page-20" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-20" id="page-20" href="#page-20" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -1138,7 +1216,8 @@ charset=UTF-8","Content-Length":106}}}
* "<a href="http://tools.ietf.org/html/rfc7233">http://tools.ietf.org/html/rfc7233</a>"
* "<a href="http://remotestorage.io/spec/web-authoring">http://remotestorage.io/spec/web-authoring</a>"
<span class="h2"><a class="selflink" name="section-16" href="#section-16">16</a>. Acknowledgements</span>
<span class="h2"><a class="selflink" name="section-16"
href="#section-16">16</a>. Acknowledgements</span>
The authors would like to thank everybody who contributed to the
development of this protocol, including Kenny Bentley, Javier Diaz,
@ -1161,7 +1240,8 @@ charset=UTF-8","Content-Length":106}}}
<span class="grey">de Jong [Page 20]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-21" id="page-21" href="#page-21" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-21" id="page-21" href="#page-21" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -1196,7 +1276,8 @@ charset=UTF-8","Content-Length":106}}}
Conditional Requests", <a href="./rfc7233">RFC7233</a>, June 2014.
[<a name="ref-SPDY" id="ref-SPDY">SPDY</a>]
Mark Belshe, Roberto Peon, "SPDY Protocol - Draft 3.1", <a href="http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1">http://</a>
Mark Belshe, Roberto Peon, "SPDY Protocol - Draft 3.1", <a
href="http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1">http://</a>
<a href="http://www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1">www.chromium.org/spdy/spdy-protocol/spdy-protocol-draft3-1</a>,
September 2013.
@ -1211,7 +1292,8 @@ charset=UTF-8","Content-Length":106}}}
<span class="grey">de Jong [Page 21]</span>
</pre><!--NewPage--><pre class='newpage'><a name="page-22" id="page-22" href="#page-22" class="invisible"> </a>
</pre><!--NewPage-->
<pre class='newpage'><a name="page-22" id="page-22" href="#page-22" class="invisible"> </a>
<span class="grey">Internet-Draft remoteStorage December 2014</span>
@ -1262,8 +1344,10 @@ charset=UTF-8","Content-Length":106}}}
de Jong [Page 22]
</pre><br />
</pre>
<br/>
<span class="noprint"><small><small>Html markup produced by rfcmarkup 1.111, available from
<a href="https://tools.ietf.org/tools/rfcmarkup/">https://tools.ietf.org/tools/rfcmarkup/</a>
</small></small></span>
</body></html>
</body>
</html>

14
app/src/main/assets/readability/test/test-pages/js-link-replacement/source.html
View File

@ -1,14 +1,14 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Replace javascript: links</title>
<meta charset="utf-8"/>
<title>Replace javascript: links</title>
</head>
<body>
<a href="javascript:">
<p>abc</p>
<p>def</p>
ghi
</a>
<a href="javascript:">
<p>abc</p>
<p>def</p>
ghi
</a>
</body>
</html>

419
app/src/main/assets/readability/test/test-pages/keep-images/expected.html
View File

@ -3,92 +3,214 @@
<div>
<figure name="b9ad" id="b9ad">
<div>
<p><img data-image-id="1*sLDnS1UWEFIS33uLMxq3cw.jpeg" data-width="2100" data-height="1402" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*sLDnS1UWEFIS33uLMxq3cw.jpeg" />
<p><img data-image-id="1*sLDnS1UWEFIS33uLMxq3cw.jpeg" data-width="2100"
data-height="1402"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*sLDnS1UWEFIS33uLMxq3cw.jpeg"/>
</p>
</div>
</figure>
</div>
<div>
<h4 name="9736" id="9736" data-align="center">Welcome to DoctorX’s Barcelona lab, where the drugs you bought online are tested for safety and purity. No questions asked.</h4>
<h4 name="9736" id="9736" data-align="center">Welcome to DoctorX’s Barcelona lab, where
the drugs you bought online are tested for safety and purity. No questions
asked.</h4>
<figure name="7417" id="7417">
<div>
<p><img data-image-id="1*3vIhkoHIzcxvUdijoCVx6w.png" data-width="1200" data-height="24" data-action="zoom" data-action-value="1*3vIhkoHIzcxvUdijoCVx6w.png" src="https://d262ilb51hltx0.cloudfront.net/max/800/1*3vIhkoHIzcxvUdijoCVx6w.png" />
<p><img data-image-id="1*3vIhkoHIzcxvUdijoCVx6w.png" data-width="1200"
data-height="24" data-action="zoom"
data-action-value="1*3vIhkoHIzcxvUdijoCVx6w.png"
src="https://d262ilb51hltx0.cloudfront.net/max/800/1*3vIhkoHIzcxvUdijoCVx6w.png"/>
</p>
</div>
</figure>
<p name="8a83" id="8a83">Standing at a table in a chemistry lab in Barcelona, Cristina Gil Lladanosa tears open a silver, smell-proof protective envelope. She slides out a transparent bag full of crystals. Around her, machines whir and hum, and other researchers mill around in long, white coats.</p>
<p name="b675" id="b675">She is holding the lab’s latest delivery of a drug bought from the “deep web,” the clandestine corner of the internet that isn’t reachable by normal search engines, and is home to some sites that require special software to access. Labeled as <a href="http://en.wikipedia.org/wiki/MDMA" data-href="http://en.wikipedia.org/wiki/MDMA" rel="nofollow">MDMA</a> (the street term is ecstasy), this sample has been shipped from Canada. Lladanosa and her colleague Iván Fornís Espinosa have also received drugs, anonymously, from people in China, Australia, Europe and the United States.</p>
<p name="3c0b" id="3c0b">“Here we have speed, MDMA, cocaine, pills,” Lladanosa says, pointing to vials full of red, green, blue and clear solutions sitting in labeled boxes.</p>
<p name="8a83" id="8a83">Standing at a table in a chemistry lab in Barcelona, Cristina
Gil Lladanosa tears open a silver, smell-proof protective envelope. She slides out a
transparent bag full of crystals. Around her, machines whir and hum, and other
researchers mill around in long, white coats.</p>
<p name="b675" id="b675">She is holding the lab’s latest delivery of a drug bought from
the “deep web,” the clandestine corner of the internet that isn’t reachable by
normal search engines, and is home to some sites that require special software to
access. Labeled as <a href="http://en.wikipedia.org/wiki/MDMA"
data-href="http://en.wikipedia.org/wiki/MDMA" rel="nofollow">MDMA</a>
(the street term is ecstasy), this sample has been shipped from Canada. Lladanosa
and her colleague Iván Fornís Espinosa have also received drugs, anonymously, from
people in China, Australia, Europe and the United States.</p>
<p name="3c0b" id="3c0b">“Here we have speed, MDMA, cocaine, pills,” Lladanosa says,
pointing to vials full of red, green, blue and clear solutions sitting in labeled
boxes.</p>
</div>
<div>
<figure name="c4e6" id="c4e6">
<div>
<p><img data-image-id="1*4gN1-fzOwCniw-DbqQjDeQ.jpeg" data-width="2100" data-height="1402" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*4gN1-fzOwCniw-DbqQjDeQ.jpeg" />
<p><img data-image-id="1*4gN1-fzOwCniw-DbqQjDeQ.jpeg" data-width="2100"
data-height="1402"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*4gN1-fzOwCniw-DbqQjDeQ.jpeg"/>
</p>
</div>
<figcaption>Cristina Gil Lladanosa, at the Barcelona testing lab | photo by Joan Bardeletti</figcaption>
<figcaption>Cristina Gil Lladanosa, at the Barcelona testing lab | photo by Joan
Bardeletti
</figcaption>
</figure>
</div>
<div>
<p name="7a54" id="7a54">Since 2011, with the launch of <a href="http://en.wikipedia.org/wiki/Silk_Road_%28marketplace%29" data-href="http://en.wikipedia.org/wiki/Silk_Road_%28marketplace%29" rel="nofollow">Silk Road</a>, anybody has been able to safely buy illegal drugs from the deep web and have them delivered to their door. Though the FBI shut down that black market in October 2013, other outlets have emerged to fill its role. For the last 10 months the lab at which Lladanosa and Espinosa work has offered a paid testing service of those drugs. By sending in samples for analysis, users can know exactly what it is they are buying, and make a more informed decision about whether to ingest the substance. The group, called <a href="http://energycontrol.org/" data-href="http://energycontrol.org/" rel="nofollow">Energy Control</a>, which has being running “harm reduction” programs since 1999, is the first to run a testing service explicitly geared towards verifying those purchases from the deep web.</p>
<p name="4395" id="4395">Before joining Energy Control, Lladanosa briefly worked at a pharmacy, whereas Espinosa spent 14 years doing drug analysis. Working at Energy Control is “more gratifying,” and “rewarding” than her previous jobs, Lladanosa told me. They also receive help from a group of volunteers, made up of a mixture of “squatters,” as Espinosa put it, and medical students, who prepare the samples for testing.</p>
<p name="0c18" id="0c18">After weighing out the crystals, aggressively mixing it with methanol until dissolved, and delicately pouring the liquid into a tiny brown bottle, Lladanosa, a petite woman who is nearly engulfed by her lab coat, is now ready to test the sample. She loads a series of three trays on top of a large white appliance sitting on a table, called a gas chromatograph (GC). A jungle of thick pipes hang from the lab’s ceiling behind it.</p>
<p name="7a54" id="7a54">Since 2011, with the launch of <a
href="http://en.wikipedia.org/wiki/Silk_Road_%28marketplace%29"
data-href="http://en.wikipedia.org/wiki/Silk_Road_%28marketplace%29"
rel="nofollow">Silk Road</a>, anybody has been able to safely buy illegal drugs
from the deep web and have them delivered to their door. Though the FBI shut down
that black market in October 2013, other outlets have emerged to fill its role. For
the last 10 months the lab at which Lladanosa and Espinosa work has offered a paid
testing service of those drugs. By sending in samples for analysis, users can know
exactly what it is they are buying, and make a more informed decision about whether
to ingest the substance. The group, called <a href="http://energycontrol.org/"
data-href="http://energycontrol.org/"
rel="nofollow">Energy Control</a>,
which has being running “harm reduction” programs since 1999, is the first to run a
testing service explicitly geared towards verifying those purchases from the deep
web.</p>
<p name="4395" id="4395">Before joining Energy Control, Lladanosa briefly worked at a
pharmacy, whereas Espinosa spent 14 years doing drug analysis. Working at Energy
Control is “more gratifying,” and “rewarding” than her previous jobs, Lladanosa told
me. They also receive help from a group of volunteers, made up of a mixture of
“squatters,” as Espinosa put it, and medical students, who prepare the samples for
testing.</p>
<p name="0c18" id="0c18">After weighing out the crystals, aggressively mixing it with
methanol until dissolved, and delicately pouring the liquid into a tiny brown
bottle, Lladanosa, a petite woman who is nearly engulfed by her lab coat, is now
ready to test the sample. She loads a series of three trays on top of a large white
appliance sitting on a table, called a gas chromatograph (GC). A jungle of thick
pipes hang from the lab’s ceiling behind it.</p>
</div>
<div>
<figure name="559c" id="559c">
<div>
<p><img data-image-id="1*2KPmZkIBUrhps-2uwDvYFQ.jpeg" data-width="2100" data-height="1402" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*2KPmZkIBUrhps-2uwDvYFQ.jpeg" />
<p><img data-image-id="1*2KPmZkIBUrhps-2uwDvYFQ.jpeg" data-width="2100"
data-height="1402"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*2KPmZkIBUrhps-2uwDvYFQ.jpeg"/>
</p>
</div>
<figcaption>Photo by Joan Bardeletti</figcaption>
</figure>
</div>
<div>
<p name="1549" id="1549">“Chromatography separates all the substances,” Lladanosa says as she loads the machine with an array of drugs sent from the deep web and local Spanish users. It can tell whether a sample is pure or contaminated, and if the latter, with what.</p>
<p name="5d0f" id="5d0f">Rushes of hot air blow across the desk as the gas chromatograph blasts the sample at 280 degrees Celsius. Thirty minutes later the machine’s robotic arm automatically moves over to grip another bottle. The machine will continue cranking through the 150 samples in the trays for most of the work week.</p>
<p name="1549" id="1549">“Chromatography separates all the substances,” Lladanosa says
as she loads the machine with an array of drugs sent from the deep web and local
Spanish users. It can tell whether a sample is pure or contaminated, and if the
latter, with what.</p>
<p name="5d0f" id="5d0f">Rushes of hot air blow across the desk as the gas chromatograph
blasts the sample at 280 degrees Celsius. Thirty minutes later the machine’s robotic
arm automatically moves over to grip another bottle. The machine will continue
cranking through the 150 samples in the trays for most of the work week.</p>
</div>
<div>
<figure name="d6aa" id="d6aa">
<div>
<p><img data-image-id="1*PU40bbbox2Ompc5I3RE99A.jpeg" data-width="2013" data-height="1241" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*PU40bbbox2Ompc5I3RE99A.jpeg" />
<p><img data-image-id="1*PU40bbbox2Ompc5I3RE99A.jpeg" data-width="2013"
data-height="1241"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*PU40bbbox2Ompc5I3RE99A.jpeg"/>
</p>
</div>
<figcaption>Photo by Joan Bardeletti</figcaption>
</figure>
</div>
<div>
<p name="15e0" id="15e0">To get the drugs to Barcelona, a user mails at least 10 milligrams of a substance to the offices of the Asociación Bienestar y Desarrollo, the non-government organization that oversees Energy Control. The sample then gets delivered to the testing service’s laboratory, at the Barcelona Biomedical Research Park, a futuristic, seven story building sitting metres away from the beach. Energy Control borrows its lab space from a biomedical research group for free.</p>
<p name="2574" id="2574">The tests cost 50 Euro per sample. Users pay, not surprisingly, with Bitcoin. In the post announcing Energy Control’s service on the deep web, the group promised that “All profits of this service are set aside of maintenance of this project.”</p>
<p name="2644" id="2644">About a week after testing, those results are sent in a PDF to an email address provided by the anonymous client.</p>
<p name="9f91" id="9f91">“The process is quite boring, because you are in a routine,” Lladanosa says. But one part of the process is consistently surprising: that moment when the results pop up on the screen. “Every time it’s something different.” For instance, one cocaine sample she had tested also contained phenacetin, a painkiller added to increase the product’s weight; lidocaine, an anesthetic that numbs the gums, giving the impression that the user is taking higher quality cocaine; and common caffeine.</p>
<p name="15e0" id="15e0">To get the drugs to Barcelona, a user mails at least 10
milligrams of a substance to the offices of the Asociación Bienestar y Desarrollo,
the non-government organization that oversees Energy Control. The sample then gets
delivered to the testing service’s laboratory, at the Barcelona Biomedical Research
Park, a futuristic, seven story building sitting metres away from the beach. Energy
Control borrows its lab space from a biomedical research group for free.</p>
<p name="2574" id="2574">The tests cost 50 Euro per sample. Users pay, not surprisingly,
with Bitcoin. In the post announcing Energy Control’s service on the deep web, the
group promised that “All profits of this service are set aside of maintenance of
this project.”</p>
<p name="2644" id="2644">About a week after testing, those results are sent in a PDF to
an email address provided by the anonymous client.</p>
<p name="9f91" id="9f91">“The process is quite boring, because you are in a routine,”
Lladanosa says. But one part of the process is consistently surprising: that moment
when the results pop up on the screen. “Every time it’s something different.” For
instance, one cocaine sample she had tested also contained phenacetin, a painkiller
added to increase the product’s weight; lidocaine, an anesthetic that numbs the
gums, giving the impression that the user is taking higher quality cocaine; and
common caffeine.</p>
<figure name="b821" id="b821">
<div>
<p><img data-image-id="1*ohyycinH18fz98TCyUzVgQ.png" data-width="1200" data-height="24" data-action="zoom" data-action-value="1*ohyycinH18fz98TCyUzVgQ.png" src="https://d262ilb51hltx0.cloudfront.net/max/800/1*ohyycinH18fz98TCyUzVgQ.png" />
<p><img data-image-id="1*ohyycinH18fz98TCyUzVgQ.png" data-width="1200"
data-height="24" data-action="zoom"
data-action-value="1*ohyycinH18fz98TCyUzVgQ.png"
src="https://d262ilb51hltx0.cloudfront.net/max/800/1*ohyycinH18fz98TCyUzVgQ.png"/>
</p>
</div>
</figure>
<p name="39a6" id="39a6">The deep web drug lab is the brainchild of Fernando Caudevilla, a Spanish physician who is better known as “DoctorX” on the deep web, a nickname given to him by his Energy Control co-workers because of his earlier writing about the history, risks and recreational culture of MDMA. In the physical world, Caudevilla has worked for over a decade with Energy Control on various harm reduction focused projects, most of which have involved giving Spanish illegal drug users medical guidance, and often writing leaflets about the harms of certain substances.</p>
<p name="39a6" id="39a6">The deep web drug lab is the brainchild of Fernando Caudevilla,
a Spanish physician who is better known as “DoctorX” on the deep web, a nickname
given to him by his Energy Control co-workers because of his earlier writing about
the history, risks and recreational culture of MDMA. In the physical world,
Caudevilla has worked for over a decade with Energy Control on various harm
reduction focused projects, most of which have involved giving Spanish illegal drug
users medical guidance, and often writing leaflets about the harms of certain
substances.</p>
</div>
<div>
<figure name="eebc" id="eebc">
<div>
<p><img data-image-id="1*mKvUNOAVQxl6atCbxbCZsg.jpeg" data-width="2100" data-height="1241" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*mKvUNOAVQxl6atCbxbCZsg.jpeg" />
<p><img data-image-id="1*mKvUNOAVQxl6atCbxbCZsg.jpeg" data-width="2100"
data-height="1241"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*mKvUNOAVQxl6atCbxbCZsg.jpeg"/>
</p>
</div>
<figcaption>Fernando Caudevilla, AKA DoctorX. Photo: Joseph Cox</figcaption>
</figure>
</div>
<div>
<p name="c099" id="c099">Caudevilla first ventured into Silk Road forums in April 2013. “I would like to contribute to this forum offering professional advice in topics related to drug use and health,” he wrote in an <a href="http://web.archive.org/web/20131015051405/https://dkn255hz262ypmii.onion.to/index.php?topic=147607.0" data-href="http://web.archive.org/web/20131015051405/https://dkn255hz262ypmii.onion.to/index.php?topic=147607.0" rel="nofollow">introductory post</a>, using his DoctorX alias. Caudevilla offered to provide answers to questions that a typical doctor is not prepared, or willing, to respond to, at least not without a lecture or a judgment. “This advice cannot replace a complete face-to-face medical evaluation,” he wrote, “but I know how difficult it can be to talk frankly about these things.”</p>
<p name="ff1d" id="ff1d">The requests flooded in. A diabetic asked what effect MDMA has on blood sugar; another what the risks of frequent psychedelic use were for a young person. Someone wanted to know whether amphetamine use should be avoided during lactation. In all, Fernando’s thread received over 50,000 visits and 300 questions before the FBI shut down Silk Road.</p>
<p name="1f35" id="1f35">“He’s amazing. A gift to this community,” one user wrote on the Silk Road 2.0 forum, a site that sprang up after the original. “His knowledge is invaluable, and never comes with any judgment.” Up until recently, Caudevilla answered questions on the marketplace “Evolution.” Last week, however, the administrators of that site <a href="http://motherboard.vice.com/read/one-of-the-darknets-biggest-markets-may-have-just-stole-all-its-users-bitcoin" data-href="http://motherboard.vice.com/read/one-of-the-darknets-biggest-markets-may-have-just-stole-all-its-users-bitcoin" rel="nofollow">pulled a scam</a>, shutting the market down and escaping with an estimated $12 million worth of Bitcoin.</p>
<p name="b20f" id="b20f">Caudevilla’s transition from dispensing advice to starting up a no-questions-asked drug testing service came as a consequence of his experience on the deep web. He’d wondered whether he could help bring more harm reduction services to a marketplace without controls. The Energy Control project, as part of its mandate of educating drug users and preventing harm, had already been carrying out drug testing for local Spanish users since 2001, at music festivals, night clubs, or through a drop-in service at a lab in Madrid.</p>
<p name="f739" id="f739">“I thought, we are doing this in Spain, why don’t we do an international drug testing service?” Caudevilla told me when I visited the other Energy Control lab, in Madrid. Caudevilla, a stocky character with ear piercings and short, shaved hair, has eyes that light up whenever he discusses the world of the deep web. Later, via email, he elaborated that it was not a hard sell. “It was not too hard to convince them,” he wrote me. Clearly, Energy Control believed that the reputation he had earned as an unbiased medical professional on the deep web might carry over to the drug analysis service, where one needs to establish “credibility, trustworthiness, [and] transparency,” Caudevilla said. “We could not make mistakes,” he added.</p>
<p name="c099" id="c099">Caudevilla first ventured into Silk Road forums in April 2013.
“I would like to contribute to this forum offering professional advice in topics
related to drug use and health,” he wrote in an <a
href="http://web.archive.org/web/20131015051405/https://dkn255hz262ypmii.onion.to/index.php?topic=147607.0"
data-href="http://web.archive.org/web/20131015051405/https://dkn255hz262ypmii.onion.to/index.php?topic=147607.0"
rel="nofollow">introductory post</a>, using his DoctorX alias. Caudevilla
offered to provide answers to questions that a typical doctor is not prepared, or
willing, to respond to, at least not without a lecture or a judgment. “This advice
cannot replace a complete face-to-face medical evaluation,” he wrote, “but I know
how difficult it can be to talk frankly about these things.”</p>
<p name="ff1d" id="ff1d">The requests flooded in. A diabetic asked what effect MDMA has
on blood sugar; another what the risks of frequent psychedelic use were for a young
person. Someone wanted to know whether amphetamine use should be avoided during
lactation. In all, Fernando’s thread received over 50,000 visits and 300 questions
before the FBI shut down Silk Road.</p>
<p name="1f35" id="1f35">“He’s amazing. A gift to this community,” one user wrote on the
Silk Road 2.0 forum, a site that sprang up after the original. “His knowledge is
invaluable, and never comes with any judgment.” Up until recently, Caudevilla
answered questions on the marketplace “Evolution.” Last week, however, the
administrators of that site <a
href="http://motherboard.vice.com/read/one-of-the-darknets-biggest-markets-may-have-just-stole-all-its-users-bitcoin"
data-href="http://motherboard.vice.com/read/one-of-the-darknets-biggest-markets-may-have-just-stole-all-its-users-bitcoin"
rel="nofollow">pulled a scam</a>, shutting the market down and escaping with
an estimated $12 million worth of Bitcoin.</p>
<p name="b20f" id="b20f">Caudevilla’s transition from dispensing advice to starting up a
no-questions-asked drug testing service came as a consequence of his experience on
the deep web. He’d wondered whether he could help bring more harm reduction services
to a marketplace without controls. The Energy Control project, as part of its
mandate of educating drug users and preventing harm, had already been carrying out
drug testing for local Spanish users since 2001, at music festivals, night clubs, or
through a drop-in service at a lab in Madrid.</p>
<p name="f739" id="f739">“I thought, we are doing this in Spain, why don’t we do an
international drug testing service?” Caudevilla told me when I visited the other
Energy Control lab, in Madrid. Caudevilla, a stocky character with ear piercings and
short, shaved hair, has eyes that light up whenever he discusses the world of the
deep web. Later, via email, he elaborated that it was not a hard sell. “It was not
too hard to convince them,” he wrote me. Clearly, Energy Control believed that the
reputation he had earned as an unbiased medical professional on the deep web might
carry over to the drug analysis service, where one needs to establish “credibility,
trustworthiness, [and] transparency,” Caudevilla said. “We could not make mistakes,”
he added.</p>
</div>
<div>
<figure name="4058" id="4058">
<div>
<p><img data-image-id="1*knT10_FNVUmqQIBLnutmzQ.jpeg" data-width="4400" data-height="3141" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*knT10_FNVUmqQIBLnutmzQ.jpeg" />
<p><img data-image-id="1*knT10_FNVUmqQIBLnutmzQ.jpeg" data-width="4400"
data-height="3141"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*knT10_FNVUmqQIBLnutmzQ.jpeg"/>
</p>
</div>
<figcaption>Photo: Joseph Cox</figcaption>
@ -97,36 +219,88 @@
<div>
<figure name="818c" id="818c">
<div>
<p><img data-image-id="1*ohyycinH18fz98TCyUzVgQ.png" data-width="1200" data-height="24" data-action="zoom" data-action-value="1*ohyycinH18fz98TCyUzVgQ.png" src="https://d262ilb51hltx0.cloudfront.net/max/800/1*ohyycinH18fz98TCyUzVgQ.png" />
<p><img data-image-id="1*ohyycinH18fz98TCyUzVgQ.png" data-width="1200"
data-height="24" data-action="zoom"
data-action-value="1*ohyycinH18fz98TCyUzVgQ.png"
src="https://d262ilb51hltx0.cloudfront.net/max/800/1*ohyycinH18fz98TCyUzVgQ.png"/>
</p>
</div>
</figure>
<p name="7b5e" id="7b5e">While the Energy Control lab in Madrid lab only tests Spanish drugs from various sources, it is the Barcelona location which vets the substances bought in the shadowy recesses of of the deep web. Caudevilla no longer runs it, having handed it over to his colleague Ana Muñoz. She maintains a presence on the deep web forums, answers questions from potential users, and sends back reports when they are ready.</p>
<p name="0f0e" id="0f0e">The testing program exists in a legal grey area. The people who own the Barcelona lab are accredited to experiment with and handle drugs, but Energy Control doesn’t have this permission itself, at least not in writing.</p>
<p name="e002" id="e002">“We have a verbal agreement with the police and other authorities. They already know what we are doing,” Lladanosa tells me. It is a pact of mutual benefit. Energy Control provides the police with information on batches of drugs in Spain, whether they’re from the deep web or not, Espinosa says. They also contribute to the European Monitoring Centre for Drugs and Drug Addiction’s early warning system, a collaboration that attempts to spread information about dangerous drugs as quickly as possible.</p>
<p name="db1b" id="db1b">By the time of my visit in February, Energy Control had received over 150 samples from the deep web and have been receiving more at a rate of between 4 and 8 a week. Traditional drugs, such as cocaine and MDMA, make up about 70 percent of the samples tested, but the Barcelona lab has also received samples of the prescription pill codeine, research chemicals and synthetic cannabinoids, and even pills of Viagra.</p>
<p name="7b5e" id="7b5e">While the Energy Control lab in Madrid lab only tests Spanish
drugs from various sources, it is the Barcelona location which vets the substances
bought in the shadowy recesses of of the deep web. Caudevilla no longer runs it,
having handed it over to his colleague Ana Muñoz. She maintains a presence on the
deep web forums, answers questions from potential users, and sends back reports when
they are ready.</p>
<p name="0f0e" id="0f0e">The testing program exists in a legal grey area. The people who
own the Barcelona lab are accredited to experiment with and handle drugs, but Energy
Control doesn’t have this permission itself, at least not in writing.</p>
<p name="e002" id="e002">“We have a verbal agreement with the police and other
authorities. They already know what we are doing,” Lladanosa tells me. It is a pact
of mutual benefit. Energy Control provides the police with information on batches of
drugs in Spain, whether they’re from the deep web or not, Espinosa says. They also
contribute to the European Monitoring Centre for Drugs and Drug Addiction’s early
warning system, a collaboration that attempts to spread information about dangerous
drugs as quickly as possible.</p>
<p name="db1b" id="db1b">By the time of my visit in February, Energy Control had
received over 150 samples from the deep web and have been receiving more at a rate
of between 4 and 8 a week. Traditional drugs, such as cocaine and MDMA, make up
about 70 percent of the samples tested, but the Barcelona lab has also received
samples of the prescription pill codeine, research chemicals and synthetic
cannabinoids, and even pills of Viagra.</p>
</div>
<div>
<figure name="b885" id="b885">
<div>
<p><img data-image-id="1*Vr61dyCTRwk6CemmVF8YAQ.jpeg" data-width="2100" data-height="1402" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*Vr61dyCTRwk6CemmVF8YAQ.jpeg" />
<p><img data-image-id="1*Vr61dyCTRwk6CemmVF8YAQ.jpeg" data-width="2100"
data-height="1402"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*Vr61dyCTRwk6CemmVF8YAQ.jpeg"/>
</p>
</div>
<figcaption>Photo by Joan Bardeletti</figcaption>
</figure>
</div>
<div>
<p name="e76f" id="e76f">So it’s fair to make a tentative judgement on what people are paying for on the deep web. The verdict thus far? Overall, drugs on the deep web appear to be of much higher quality than those found on the street.</p>
<p name="5352" id="5352">“In general, the cocaine is amazing,” says Caudevilla, saying that the samples they’ve seen have purities climbing towards 80 or 90 percent, and some even higher. To get an idea of how unusual this is, take a look at the <a href="http://www.unodc.org/documents/wdr2014/Cocaine_2014_web.pdf" data-href="http://www.unodc.org/documents/wdr2014/Cocaine_2014_web.pdf" rel="nofollow">United Nations Office on Drugs and Crime World Drug Report 2014</a>, which reports that the average quality of street cocaine in Spain is just over 40 percent, while in the United Kingdom it is closer to 30 percent.“We have found 100 percent [pure] cocaine,” he adds. “That’s really, really strange. That means that, technically, this cocaine has been purified, with clandestine methods.”</p>
<p name="a71c" id="a71c">Naturally, identifying vendors who sell this top-of-the-range stuff is one of the reasons that people have sent samples to Energy Control. Caudevilla was keen to stress that, officially, Energy Control’s service “is not intended to be a control of drug quality,” meaning a vetting process for identifying the best sellers, but that is exactly how some people have been using it.</p>
<p name="cb5b" id="cb5b">As one buyer on the Evolution market, elmo666, wrote to me over the site’s messaging system, “My initial motivations were selfish. My primary motivation was to ensure that I was receiving and continue to receive a high quality product, essentially to keep the vendor honest as far as my interactions with them went.”</p>
<p name="d80d" id="d80d">Vendors on deep web markets advertise their product just like any other outlet does, using flash sales, gimmicky giveaways and promises of drugs that are superior to those of their competitors. The claims, however, can turn out to be empty: despite the test results that show that deep web cocaine vendors typically sell product that is of a better quality than that found on the street, in plenty of cases, the drugs are nowhere near as pure as advertised.</p>
<p name="36de" id="36de">“You won’t be getting anything CLOSE to what you paid for,” one user complained about the cocaine from ‘Mirkov’, a vendor on Evolution. “He sells 65% not 95%.”</p>
<p name="e76f" id="e76f">So it’s fair to make a tentative judgement on what people are
paying for on the deep web. The verdict thus far? Overall, drugs on the deep web
appear to be of much higher quality than those found on the street.</p>
<p name="5352" id="5352">“In general, the cocaine is amazing,” says Caudevilla, saying
that the samples they’ve seen have purities climbing towards 80 or 90 percent, and
some even higher. To get an idea of how unusual this is, take a look at the <a
href="http://www.unodc.org/documents/wdr2014/Cocaine_2014_web.pdf"
data-href="http://www.unodc.org/documents/wdr2014/Cocaine_2014_web.pdf"
rel="nofollow">United Nations Office on Drugs and Crime World Drug Report
2014</a>, which reports that the average quality of street cocaine in Spain is
just over 40 percent, while in the United Kingdom it is closer to 30 percent.“We
have found 100 percent [pure] cocaine,” he adds. “That’s really, really strange.
That means that, technically, this cocaine has been purified, with clandestine
methods.”</p>
<p name="a71c" id="a71c">Naturally, identifying vendors who sell this top-of-the-range
stuff is one of the reasons that people have sent samples to Energy Control.
Caudevilla was keen to stress that, officially, Energy Control’s service “is not
intended to be a control of drug quality,” meaning a vetting process for identifying
the best sellers, but that is exactly how some people have been using it.</p>
<p name="cb5b" id="cb5b">As one buyer on the Evolution market, elmo666, wrote to me over
the site’s messaging system, “My initial motivations were selfish. My primary
motivation was to ensure that I was receiving and continue to receive a high quality
product, essentially to keep the vendor honest as far as my interactions with them
went.”</p>
<p name="d80d" id="d80d">Vendors on deep web markets advertise their product just like
any other outlet does, using flash sales, gimmicky giveaways and promises of drugs
that are superior to those of their competitors. The claims, however, can turn out
to be empty: despite the test results that show that deep web cocaine vendors
typically sell product that is of a better quality than that found on the street, in
plenty of cases, the drugs are nowhere near as pure as advertised.</p>
<p name="36de" id="36de">“You won’t be getting anything CLOSE to what you paid for,” one
user complained about the cocaine from ‘Mirkov’, a vendor on Evolution. “He sells
65% not 95%.”</p>
</div>
<div>
<figure name="8544" id="8544">
<div>
<p><img data-image-id="1*a-1_13xE6_ErQ-QSlz6myw.jpeg" data-width="2100" data-height="1402" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*a-1_13xE6_ErQ-QSlz6myw.jpeg" />
<p><img data-image-id="1*a-1_13xE6_ErQ-QSlz6myw.jpeg" data-width="2100"
data-height="1402"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*a-1_13xE6_ErQ-QSlz6myw.jpeg"/>
</p>
</div>
<figcaption>Photo by Joan Bardeletti</figcaption>
@ -135,72 +309,183 @@
<div>
<figure name="d521" id="d521">
<div>
<p><img data-image-id="1*ohyycinH18fz98TCyUzVgQ.png" data-width="1200" data-height="24" data-action="zoom" data-action-value="1*ohyycinH18fz98TCyUzVgQ.png" src="https://d262ilb51hltx0.cloudfront.net/max/800/1*ohyycinH18fz98TCyUzVgQ.png" />
<p><img data-image-id="1*ohyycinH18fz98TCyUzVgQ.png" data-width="1200"
data-height="24" data-action="zoom"
data-action-value="1*ohyycinH18fz98TCyUzVgQ.png"
src="https://d262ilb51hltx0.cloudfront.net/max/800/1*ohyycinH18fz98TCyUzVgQ.png"/>
</p>
</div>
</figure>
<p name="126b" id="126b">Despite the prevalence of people using the service to gauge the quality of what goes up their nose, many users send samples to Energy Control in the spirit of its original mission: keeping themselves alive and healthy. The worst case scenario from drugs purchased on the deep web is, well the worst case. That was the outcome when <a href="http://www.independent.co.uk/news/uk/crime/teenager-patrick-mcmullen-who-died-while-on-skype-had-bought-drugs-from-silk-road-8942329.html" data-href="http://www.independent.co.uk/news/uk/crime/teenager-patrick-mcmullen-who-died-while-on-skype-had-bought-drugs-from-silk-road-8942329.html" rel="nofollow">Patrick McMullen,</a> a 17-year-old Scottish student, ingested half a gram of MDMA and three tabs of LSD, reportedly purchased from the Silk Road. While talking to his friends on Skype, his words became slurred and he passed out. Paramedics could not revive him. The coroner for that case, Sherrif Payne, who deemed the cause of death ecstasy toxicity, told <em>The Independent</em> “You never know the purity of what you are taking and you can easily come unstuck.”</p>
<p name="5e9e" id="5e9e">ScreamMyName, a deep web user who has been active since the original Silk Road, wants to alert users to the dangerous chemicals that are often mixed with drugs, and is using Energy Control as a means to do so.</p>
<p name="19a6" id="19a6">“We’re at a time where some vendors are outright sending people poison. Some do it unknowingly,” ScreamMyName told me in an encrypted message. “Cocaine production in South America is often tainted with either levamisole or phenacetine. Both poison to humans and both with severe side effects.”</p>
<p name="9fef" id="9fef">In the case of Levamisole, those prescribing it are often not doctors but veterinarians, as Levamisole is commonly used on animals, primarily for the treatment of worms. If ingested by humans it can lead to cases of extreme eruptions of the skin, as <a href="http://www.ncbi.nlm.nih.gov/pubmed/22127712" data-href="http://www.ncbi.nlm.nih.gov/pubmed/22127712" rel="nofollow">documented in a study</a> from researchers at the University of California, San Francisco. But Lladanosa has found Levamisole in cocaine samples; dealers use it to increase the product weight, allowing them to stretch their batch further for greater profit — and also, she says, because Levamisole has a strong stimulant effect.</p>
<p name="7886" id="7886">“It got me sick as fuck,” Dr. Feel, an Evolution user, wrote on the site’s forums after consuming cocaine that had been cut with 23 percent Levamisole, and later tested by Energy Control. “I was laid up in bed for several days because of that shit. The first night I did it, I thought I was going to die. I nearly drove myself to the ER.”</p>
<p name="18d3" id="18d3">“More people die because of tainted drugs than the drugs themselves,” Dr. Feel added. “It’s the cuts and adulterants that are making people sick and killing them.”</p>
<p name="126b" id="126b">Despite the prevalence of people using the service to gauge the
quality of what goes up their nose, many users send samples to Energy Control in the
spirit of its original mission: keeping themselves alive and healthy. The worst case
scenario from drugs purchased on the deep web is, well the worst case. That was the
outcome when <a
href="http://www.independent.co.uk/news/uk/crime/teenager-patrick-mcmullen-who-died-while-on-skype-had-bought-drugs-from-silk-road-8942329.html"
data-href="http://www.independent.co.uk/news/uk/crime/teenager-patrick-mcmullen-who-died-while-on-skype-had-bought-drugs-from-silk-road-8942329.html"
rel="nofollow">Patrick McMullen,</a> a 17-year-old Scottish student,
ingested half a gram of MDMA and three tabs of LSD, reportedly purchased from the
Silk Road. While talking to his friends on Skype, his words became slurred and he
passed out. Paramedics could not revive him. The coroner for that case, Sherrif
Payne, who deemed the cause of death ecstasy toxicity, told <em>The Independent</em>
“You never know the purity of what you are taking and you can easily come unstuck.”
</p>
<p name="5e9e" id="5e9e">ScreamMyName, a deep web user who has been active since the
original Silk Road, wants to alert users to the dangerous chemicals that are often
mixed with drugs, and is using Energy Control as a means to do so.</p>
<p name="19a6" id="19a6">“We’re at a time where some vendors are outright sending people
poison. Some do it unknowingly,” ScreamMyName told me in an encrypted message.
“Cocaine production in South America is often tainted with either levamisole or
phenacetine. Both poison to humans and both with severe side effects.”</p>
<p name="9fef" id="9fef">In the case of Levamisole, those prescribing it are often not
doctors but veterinarians, as Levamisole is commonly used on animals, primarily for
the treatment of worms. If ingested by humans it can lead to cases of extreme
eruptions of the skin, as <a href="http://www.ncbi.nlm.nih.gov/pubmed/22127712"
data-href="http://www.ncbi.nlm.nih.gov/pubmed/22127712"
rel="nofollow">documented in a study</a> from
researchers at the University of California, San Francisco. But Lladanosa has found
Levamisole in cocaine samples; dealers use it to increase the product weight,
allowing them to stretch their batch further for greater profit — and also, she
says, because Levamisole has a strong stimulant effect.</p>
<p name="7886" id="7886">“It got me sick as fuck,” Dr. Feel, an Evolution user, wrote on
the site’s forums after consuming cocaine that had been cut with 23 percent
Levamisole, and later tested by Energy Control. “I was laid up in bed for several
days because of that shit. The first night I did it, I thought I was going to die. I
nearly drove myself to the ER.”</p>
<p name="18d3" id="18d3">“More people die because of tainted drugs than the drugs
themselves,” Dr. Feel added. “It’s the cuts and adulterants that are making people
sick and killing them.”</p>
</div>
<div>
<figure name="552a" id="552a">
<div>
<p><img data-image-id="1*IWXhtSsVv0gNnCwnDEXk-Q.jpeg" data-width="2100" data-height="1192" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*IWXhtSsVv0gNnCwnDEXk-Q.jpeg" />
<p><img data-image-id="1*IWXhtSsVv0gNnCwnDEXk-Q.jpeg" data-width="2100"
data-height="1192"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*IWXhtSsVv0gNnCwnDEXk-Q.jpeg"/>
</p>
</div>
<figcaption>Photo by Joan Bardeletti</figcaption>
</figure>
</div>
<div>
<p name="839a" id="839a">The particular case of cocaine cut with Levamisole is one of the reasons that ScreamMyName has been pushing for more drug testing on the deep web markets. “I recognize that drug use isn’t exactly healthy, but why exacerbate the problem?” he told me when I contacted him after his post. “[Energy Control] provides a way for users to test the drugs they’ll use and for these very users to know what it is they’re putting in their bodies. Such services are in very short supply.”</p>
<p name="18dc" id="18dc">After sending a number of Energy Control tests himself, ScreamMyName started a de facto crowd-sourcing campaign to get more drugs sent to the lab, and then shared the results, after throwing in some cash to get the ball rolling. <a href="https://blockchain.info/address/1Mi6VjMFqjcD48FPV7cnPB24MAtQQenRy3" data-href="https://blockchain.info/address/1Mi6VjMFqjcD48FPV7cnPB24MAtQQenRy3" rel="nofollow">He set up a Bitcoin wallet</a>, with the hope that users might chip in to fund further tests. At the time of writing, the wallet has received a total of 1.81 bitcoins; around $430 at today’s exchange rates.</p>
<p name="dcbd" id="dcbd">In posts to the Evolution community, ScreamMyName pitched this project as something that will benefit users and keep drug dealer honest. “When the funds build up to a point where we can purchase an [Energy Control] test fee, we’ll do a US thread poll for a few days and try to cohesively decide on what vendor to test,” he continued.</p>
<p name="839a" id="839a">The particular case of cocaine cut with Levamisole is one of
the reasons that ScreamMyName has been pushing for more drug testing on the deep web
markets. “I recognize that drug use isn’t exactly healthy, but why exacerbate the
problem?” he told me when I contacted him after his post. “[Energy Control] provides
a way for users to test the drugs they’ll use and for these very users to know what
it is they’re putting in their bodies. Such services are in very short supply.”</p>
<p name="18dc" id="18dc">After sending a number of Energy Control tests himself,
ScreamMyName started a de facto crowd-sourcing campaign to get more drugs sent to
the lab, and then shared the results, after throwing in some cash to get the ball
rolling. <a
href="https://blockchain.info/address/1Mi6VjMFqjcD48FPV7cnPB24MAtQQenRy3"
data-href="https://blockchain.info/address/1Mi6VjMFqjcD48FPV7cnPB24MAtQQenRy3"
rel="nofollow">He set up a Bitcoin wallet</a>, with the hope that users
might chip in to fund further tests. At the time of writing, the wallet has received
a total of 1.81 bitcoins; around $430 at today’s exchange rates.</p>
<p name="dcbd" id="dcbd">In posts to the Evolution community, ScreamMyName pitched this
project as something that will benefit users and keep drug dealer honest. “When the
funds build up to a point where we can purchase an [Energy Control] test fee, we’ll
do a US thread poll for a few days and try to cohesively decide on what vendor to
test,” he continued.</p>
</div>
<div>
<figure name="9d32" id="9d32">
<div>
<p><img data-image-id="1*NGcrjfkV0l37iQH2uyYjEw.jpeg" data-width="1368" data-height="913" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*NGcrjfkV0l37iQH2uyYjEw.jpeg" />
<p><img data-image-id="1*NGcrjfkV0l37iQH2uyYjEw.jpeg" data-width="1368"
data-height="913"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*NGcrjfkV0l37iQH2uyYjEw.jpeg"/>
</p>
</div>
<figcaption>Photo by Joan Bardeletti</figcaption>
</figure>
</div>
<div>
<p name="bff6" id="bff6">Other members of the community have been helping out, too. PlutoPete, a vendor from the original Silk Road who sold cannabis seeds and other legal items, has provided ScreamMyName with packaging to safely send the samples to Barcelona. “A box of baggies, and a load of different moisture barrier bags,” PlutoPete told me over the phone. “That’s what all the vendors use.”</p>
<p name="bb78" id="bb78">It’s a modest program so far. ScreamMyName told me that so far he had gotten enough public funding to purchase five different Energy Control tests, in addition to the ten or so he’s sent himself so far. “The program created is still in its infancy and it is growing and changing as we go along but I have a lot of faith in what we’re doing,” he says.</p>
<p name="5638" id="5638">But the spirit is contagious: elmo666, the other deep web user testing cocaine, originally kept the results of the drug tests to himself, but he, too, saw a benefit to distributing the data. “It is clear that it is a useful service to other users, keeping vendors honest and drugs (and their users) safe,” he told me. He started to report his findings to others on the forums, and then created a thread with summaries of the test results, as well as comments from the vendors if they provided it. Other users were soon basing their decisions on what to buy on elmo666‘s tests.</p>
<p name="de75" id="de75">“I’m defo trying the cola based on the incredibly helpful elmo and his energy control results and recommendations,” wrote user jayk1984. On top of this, elmo666 plans to launch an independent site on the deep web that will collate all of these results, which should act as a resource for users of all the marketplaces.</p>
<p name="6b72" id="6b72">As word of elmo666's efforts spread, he began getting requests from drug dealers who wanted him to use their wares for testing. Clearly, they figured that a positive result from Energy Control would be a fantastic marketing tool to draw more customers. They even offered elmo666 free samples. (He passed.)</p>
<p name="b008" id="b008">Meanwhile, some in the purchasing community are arguing that those running markets on the deep web should be providing quality control themselves. PlutoPete told me over the phone that he had been in discussions about this with Dread Pirate Roberts, the pseudonymous owner of the original Silk Road site. “We [had been] talking about that on a more organized basis on Silk Road 1, doing lots of anonymous buys to police each category. But of course they took the thing [Silk Road] down before we got it properly off the ground,” he lamented.</p>
<p name="49c8" id="49c8">But perhaps it is best that the users, those who are actually consuming the drugs, remain in charge of shaming dealers and warning each other. “It’s our responsibility to police the market based on reviews and feedback,” elmo666 wrote in an Evolution forum post. It seems that in the lawless space of the deep web, where everything from child porn to weapons are sold openly, users have cooperated in an organic display of self-regulation to stamp out those particular batches of drugs that are more likely to harm users.</p>
<p name="386d" id="386d">“That’s always been the case with the deep web,” PlutoPete told me. Indeed, ever since Silk Road, a stable of the drug markets has been the review system, where buyers can leave a rating and feedback for vendors, letting others know about the reliability of the seller. But DoctorX’s lab, rigorously testing the products with scientific instruments, takes it a step further.</p>
<p name="bff6" id="bff6">Other members of the community have been helping out, too.
PlutoPete, a vendor from the original Silk Road who sold cannabis seeds and other
legal items, has provided ScreamMyName with packaging to safely send the samples to
Barcelona. “A box of baggies, and a load of different moisture barrier bags,”
PlutoPete told me over the phone. “That’s what all the vendors use.”</p>
<p name="bb78" id="bb78">It’s a modest program so far. ScreamMyName told me that so far
he had gotten enough public funding to purchase five different Energy Control tests,
in addition to the ten or so he’s sent himself so far. “The program created is still
in its infancy and it is growing and changing as we go along but I have a lot of
faith in what we’re doing,” he says.</p>
<p name="5638" id="5638">But the spirit is contagious: elmo666, the other deep web user
testing cocaine, originally kept the results of the drug tests to himself, but he,
too, saw a benefit to distributing the data. “It is clear that it is a useful
service to other users, keeping vendors honest and drugs (and their users) safe,” he
told me. He started to report his findings to others on the forums, and then created
a thread with summaries of the test results, as well as comments from the vendors if
they provided it. Other users were soon basing their decisions on what to buy on
elmo666‘s tests.</p>
<p name="de75" id="de75">“I’m defo trying the cola based on the incredibly helpful elmo
and his energy control results and recommendations,” wrote user jayk1984. On top of
this, elmo666 plans to launch an independent site on the deep web that will collate
all of these results, which should act as a resource for users of all the
marketplaces.</p>
<p name="6b72" id="6b72">As word of elmo666's efforts spread, he began getting requests
from drug dealers who wanted him to use their wares for testing. Clearly, they
figured that a positive result from Energy Control would be a fantastic marketing
tool to draw more customers. They even offered elmo666 free samples. (He
passed.)</p>
<p name="b008" id="b008">Meanwhile, some in the purchasing community are arguing that
those running markets on the deep web should be providing quality control
themselves. PlutoPete told me over the phone that he had been in discussions about
this with Dread Pirate Roberts, the pseudonymous owner of the original Silk Road
site. “We [had been] talking about that on a more organized basis on Silk Road 1,
doing lots of anonymous buys to police each category. But of course they took the
thing [Silk Road] down before we got it properly off the ground,” he lamented.</p>
<p name="49c8" id="49c8">But perhaps it is best that the users, those who are actually
consuming the drugs, remain in charge of shaming dealers and warning each other.
“It’s our responsibility to police the market based on reviews and feedback,”
elmo666 wrote in an Evolution forum post. It seems that in the lawless space of the
deep web, where everything from child porn to weapons are sold openly, users have
cooperated in an organic display of self-regulation to stamp out those particular
batches of drugs that are more likely to harm users.</p>
<p name="386d" id="386d">“That’s always been the case with the deep web,” PlutoPete told
me. Indeed, ever since Silk Road, a stable of the drug markets has been the review
system, where buyers can leave a rating and feedback for vendors, letting others
know about the reliability of the seller. But DoctorX’s lab, rigorously testing the
products with scientific instruments, takes it a step further.</p>
</div>
<div>
<figure name="890b" id="890b">
<div>
<p><img data-image-id="1*WRlKt3q3mt7utmwxcbl3sQ.jpeg" data-width="2100" data-height="1373" src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*WRlKt3q3mt7utmwxcbl3sQ.jpeg" />
<p><img data-image-id="1*WRlKt3q3mt7utmwxcbl3sQ.jpeg" data-width="2100"
data-height="1373"
src="https://d262ilb51hltx0.cloudfront.net/max/2000/1*WRlKt3q3mt7utmwxcbl3sQ.jpeg"/>
</p>
</div>
<figcaption>Photo by Joan Bardeletti</figcaption>
</figure>
</div>
<div>
<p name="b109" id="b109">“In the white market, they have quality control. In the dark market, it should be the same,” Cristina Gil Lladanosa says to me before I leave the Barcelona lab.</p>
<p name="e3a4" id="e3a4">A week after I visit the lab, the results of the MDMA arrive in my inbox: it is 85 percent pure, with no indications of other active ingredients. Whoever ordered that sample from the digital shelves of the deep web, and had it shipped to their doorstep in Canada, got hold of some seriously good, and relatively safe drugs. And now they know it.</p>
<p name="b109" id="b109">“In the white market, they have quality control. In the dark
market, it should be the same,” Cristina Gil Lladanosa says to me before I leave the
Barcelona lab.</p>
<p name="e3a4" id="e3a4">A week after I visit the lab, the results of the MDMA arrive in
my inbox: it is 85 percent pure, with no indications of other active ingredients.
Whoever ordered that sample from the digital shelves of the deep web, and had it
shipped to their doorstep in Canada, got hold of some seriously good, and relatively
safe drugs. And now they know it.</p>
<figure name="31cf" id="31cf">
<div>
<p><img data-image-id="1*320_4I0lxbn5x3bx4XPI5Q.png" data-width="1200" data-height="24" data-action="zoom" data-action-value="1*320_4I0lxbn5x3bx4XPI5Q.png" src="https://d262ilb51hltx0.cloudfront.net/max/800/1*320_4I0lxbn5x3bx4XPI5Q.png" />
<p><img data-image-id="1*320_4I0lxbn5x3bx4XPI5Q.png" data-width="1200"
data-height="24" data-action="zoom"
data-action-value="1*320_4I0lxbn5x3bx4XPI5Q.png"
src="https://d262ilb51hltx0.cloudfront.net/max/800/1*320_4I0lxbn5x3bx4XPI5Q.png"/>
</p>
</div>
</figure>
<p name="9b87" id="9b87" data-align="center"><em>Top photo by Joan Bardeletti</em>
</p>
<p name="c30a" id="c30a" data-align="center">Follow Backchannel: <a href="https://twitter.com/backchnnl" data-href="https://twitter.com/backchnnl" rel="nofollow"><em>Twitter</em></a>
<em>|</em><a href="https://www.facebook.com/pages/Backchannel/1488568504730671" data-href="https://www.facebook.com/pages/Backchannel/1488568504730671" rel="nofollow"><em>Facebook</em></a>
<p name="c30a" id="c30a" data-align="center">Follow Backchannel: <a
href="https://twitter.com/backchnnl" data-href="https://twitter.com/backchnnl"
rel="nofollow"><em>Twitter</em></a>
<em>|</em><a href="https://www.facebook.com/pages/Backchannel/1488568504730671"
data-href="https://www.facebook.com/pages/Backchannel/1488568504730671"
rel="nofollow"><em>Facebook</em></a>
</p>
</div>
</div>

2138
app/src/main/assets/readability/test/test-pages/keep-images/source.html
View File

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More