From b415d1d37cc072d3dd88355002754e7658ad96bf Mon Sep 17 00:00:00 2001 From: Janet Date: Thu, 1 Dec 2016 20:05:15 -0500 Subject: [PATCH] feat: aol custom extractor (#42) * feat: aol custom parser * removed work from other commits. merged with latest master --- fixtures/www.aol.com/1480618816916.html | 5 + .../www.theguardian.com/1480457558008.html | 70 ++++++++++++ src/extractors/custom/index.js | 1 + src/extractors/custom/www.aol.com/index.js | 53 +++++++++ .../custom/www.aol.com/index.test.js | 106 ++++++++++++++++++ 5 files changed, 235 insertions(+) create mode 100644 fixtures/www.aol.com/1480618816916.html create mode 100644 fixtures/www.theguardian.com/1480457558008.html create mode 100644 src/extractors/custom/www.aol.com/index.js create mode 100644 src/extractors/custom/www.aol.com/index.test.js diff --git a/fixtures/www.aol.com/1480618816916.html b/fixtures/www.aol.com/1480618816916.html new file mode 100644 index 000000000..7d1ea873b --- /dev/null +++ b/fixtures/www.aol.com/1480618816916.html @@ -0,0 +1,5 @@ + Son of slain police officer given teddy bears made from dad's uniform - AOL News

Son of slain police officer given teddy bears made from dad's uniform

Before you go, we thought you'd like these...
Before you go close icon
Recommendation image
READ MORE
Recommendation image
READ MORE
Recommendation image
READ MORE
Dec 1st 2016 1:01PM

ST. LOUIS, MO (KTVI) – Amid unimaginable grief, the widow of slain Saint Louis County Police Officer Blake Snyder is using Facebook to help her deal with the loss of her husband. Although Elizabeth Snyder still endures immense pain, their son Malachi has helped her push through the heartache and traumatic loss.

Elizabeth posted a picture Tuesday evening of her son with what appear to be two teddy bears. She writes, "These beautiful bears were made for us from Blake's uniforms."

Elizabeth has said in the past that she's eternally grateful for all of comfort she has received from the St. Louis community and across the nation as well, adding her son will always know how much the community adored her husband and cared for their family in this trying time.

This is another image Elizabeth Snyder posted to Facebook on Sunday. It says, "I cannot describe what this necklace means to me. Last night I had dinner with a couple amazing friends, and when we were about to leave, Kristen hands me an envelope and tells me not to open until I get home. Well, of course I had to open it right then. She found this necklace in one of her purses and remembered that I had asked her to hold it one night because I didn't have pockets. This was years ago! I thought I had lost it forever. Blake had it specially made for me, so when I lost it, we were both devastated. I just really cannot describe what it means to me. It's the little things like this that keep tearing my heart apart. I miss him so very much."

This is Elizabeth Snyder's Facebook post from Thanksgiving. She writes,"Today I am thankful for, above all else, Blake and Malachi. My life was forever changed the day I started dating Blake Curtis. His love, kindness, generosity, and overall goofy personality won me over before I knew it. The last 6 years of my life spent with him showed me just how deep love can be and just how fast it can be taken away. The moment Blake saw Malachi for the first time will always be a memory I will cherish and hold on to. He wasn't perfect, (no one is), but he WAS a perfect father. He loved Chi with every fiber of his being and wanted the world for him. These two boys mean everything to me, and I'm thankful I have a bit of Blake still here with me. I am also thankful for every person who has generously helped support Chi and I. I'm not sure where we would be without you all. Thank you, thank you, thank you! I pray you all have a Happy Thanksgiving!"

Officer Snyder and his partner were responding to a disturbance call in the early morning hours on October 6 at a home in the 10700 block of Arno Drive in Green Park. The accused shooter, Trent Forster, was seated inside a car parked outside the residence when Snyder and the other officer approached. Forster allegedly shot Snyder when the officer asked to see his hands. Snyder's partner returned fire, hitting Forster several times. Forster was hospitalized and has since been charged with first-degree murder and armed criminal action.

Blake Snyder was laid to rest Thursday, October 13 following a service at St. Louis Family Church in Chesterfield.

The BackStoppers organization is continuing to raise money across the region for Officer Snyder's family.

See photos from this story:

4 PHOTOS
Son of slain police officer given teddy bears made from dad's uniform
See Gallery
Son of slain police officer given teddy bears made from dad's uniform
of
SEE ALL
BACK TO SLIDE
SHOW CAPTION +
HIDE CAPTION –

Read Full Story

Sign up for Breaking News by AOL to get the latest breaking news alerts and updates delivered straight to your inbox.

Subscribe to our other newsletters

Emails may offer personalized content or ads. Learn more. You may unsubscribe any time.

From Our Partners

Blank Blank Congress Gives Homeowners Who Owe Less Than $300-625K a Once-in-a-Lifetime Mortgage Bailout LowerMyBills
The Latest from our Partners
1 - 6 of 15
LendingTree
NextAdvisor
LowerMyBills
GoBankingRates
Boredom Therapy
Autoblog
Thu, Dec 01
Set Your Location
City, State, or Zip
\ No newline at end of file diff --git a/fixtures/www.theguardian.com/1480457558008.html b/fixtures/www.theguardian.com/1480457558008.html new file mode 100644 index 000000000..ca11be8e3 --- /dev/null +++ b/fixtures/www.theguardian.com/1480457558008.html @@ -0,0 +1,70 @@ + Standing Rock protesters hold out against extraordinary police violence | US news | The Guardian
Dakota Access pipeline

Standing Rock protesters hold out against extraordinary police violence

Apprehension and distrust pervade North Dakota protest site as promises from state that there are no plans to forcibly remove people does little to assuage fears

Police blast Standing Rock protesters with water cannon and rubber bullets – video

and

Police violence against Standing Rock protesters in North Dakota has risen to extraordinary levels, and activists and observers fear that, with two evacuation orders looming, the worst is yet to come.

A litany of munitions, including water cannons, combined with ambiguous government leadership and misleading police statements, have resulted in mass arrests, serious injuries and a deeply sown atmosphere of fear and distrust on the banks of the Missouri river.

Statements by the US Army Corps of Engineers and North Dakota state government that, despite their orders of evacuation, there are no plans to forcibly remove protesters opposing the Dakota Access pipeline have done little to assuage fears.

As the first snows have fallen and more protesters arrive in support, apprehension at the encampments about the coming days is running high.

“We’re going to hope for the absolute best,” said Linda Black Elk, a member of the Catawba Nation who works with the Standing Rock Medic & Healer Council. “If they do attempt to remove people forcibly, we are certainly preparing for mass casualties.”

‘An infinite arsenal’

Standing Rock protests
Pinterest
Police use a water cannon on Standing Rock protesters last week. Photograph: Stephanie Keith/Reuters

Harkening back to an earlier era, when police in Birmingham, Alabama, attacked African American schoolchildren with dogs and high-pressure water hoses, North Dakota officers trained water cannons on hundreds of Dakota Access pipeline protesters.

On the night of 20 November, though, the temperature was below freezing and the protesters, who call themselves “water protectors”, were camping outdoors for the evening.

Water is just one many “less-than-lethal” munitions that have been trained against the activists.

“They seem to have almost an infinite arsenal of different types of weapons,” said Rachel Lederman, attorney for the National Lawyers Guild (NLG). “I don’t think local law enforcement understands how dangerous they are.”

Police have acknowledged using sponge rounds, bean bag rounds, stinger rounds, teargas grenades, pepper spray, Mace, Tasers and a sound weapon. The explosive teargas grenades in use at Standing Rock have been banned by some US law enforcement agencies because they indiscriminately spray people, Lederman said.

“I feel like Morton County law enforcement is experimenting on us,” Black Elk said. “It’s like they received all this free military equipment and they’re just itching to try it out.”

On Monday, the NLG filed a federal class action lawsuit against the Morton County sheriff’s office, alleging excessive force at the 20 November encounter. The group is seeking a temporary restraining order to bar police from using “less-than-lethal” munitions, chemical agents, sound cannons, and water against activists.

More than two dozen people were hospitalized and 300 injured during the conflict, according to the medic and healer council. One woman’s arm was nearly blown off, according to her father, and the complaint alleges that another woman was shot in the eye, resulting in the detachment of her retina and possible permanent blindness.

“It is only a matter of luck that no one has been killed,” Lederman said about the suit. “This must stop.”

‘A messy, messy mess’

Police
Pinterest
Police stand guard during a prayer ceremony on Backwater Bridge on Sunday. Photograph: Stephanie Keith/Reuters

North Dakota law enforcement officials have released misleading or blatantly inaccurate statements about protest actions while using charged language to paint peaceful demonstrators as violent threats to public safety.

“There’s a lot of testimony on the part of the police that doesn’t match up with what we are seeing,” said Roberto Borrero, an International Indian Treaty Council representative who assisted the United Nations in an investigation of human rights abuses by law enforcement against Standing Rock activists.

When asked about the water cannons, a sheriff’s spokeswoman initially told the Guardian that protesters were “starting fires”. But footage later emerged showing police aiming the hoses directly at activists. The department subsequently conceded that it had purposefully used water on the protesters.

On Thanksgiving, Morton County sheriff Kyle Kirchmeier released a statement condemning the actions of “paid agitators and protesters” without offering any evidence that people were being paid to fight the pipeline. The department has not responded to requests to substantiate the claim.

In another statement that week, the sheriff said activists were not engaged in “civil disobedience” but were acting like “evil agitators”. The Mandan, North Dakota, police chief, Jason Ziegler, has asserted that law enforcement agencies “can use whatever force necessary to maintain peace”.

Caro Gonzales, a 26-year-old member of the Chemehuevi tribe and an anti-police violence activist in Washington state, said the language from law enforcement officials resembled that used to discredit unarmed black men killed by police.

Ferguson police officer Darren Wilson, who fatally shot Michael Brown in 2014, said the unarmed black teenager looked like a “demon”.

“Police and police unions use this language so they can demonize people who are fighting for their rights,” said Gonzales.

A lack of transparency, confusing jurisdictions, and the fact that dozens of law enforcement agencies have assisted the local sheriff’s department further complicate the situation.

The Morton County sheriff’s department has refused to reveal which law enforcement agencies were involved in the 20 November action. But the American Civil Liberties Union (ACLU) has assembled a list of 76 different agencies – state, county, and local agencies from North Dakota and nine other states – that have sent officers to the protests.

“There are real concerns about how the Morton County sheriff’s department is exercising command and control,” said Jennifer Cook, policy director for the ACLU of North Dakota. “Who makes the decision about how to police the protest? When you have so many agencies involved, each one is going to have its own training and protocol.”

Adding to the confusion is an exceedingly complicated jurisdictional terrain.

The first Standing Rock encampment, known as Sacred Stone, was established in April on land owned by LaDonna Brave Bull Allard, which sits within the boundaries of the Standing Rock Sioux reservation.

But as more activists joined the movement, camps were established north of the reservation border, on land managed by the Army Corps of Engineers and leased to a private rancher. While the government asserts that the land falls under its jurisdiction, the Standing Rock Sioux tribe says that it has treaty rights dating from the 1851 Treaty of Fort Laramie.

Given the complexity, Ronal Serpas, a retired police chief and professor of criminology at Loyola University, said the courts needed to step in and provide direction for law enforcement officers.

“This is what you would call a messy, messy mess,” Serpas said. “It’s unfair for the police to be at the center of that storm.”

‘It’s just under the surface’

Standing Rock protest
Pinterest
Women hold a prayer ceremony during a protest against plans to pass the Dakota Access pipeline near the Standing Rock reservation. Photograph: Stephanie Keith/Reuters

Many Native Americans involved in the Standing Rock movement ascribe the harsh police tactics to intense racism against indigenous people, both from law enforcement officials and civilians.

Glenn Morris, spokesman for the leadership council of the American Indian Movement of Colorado, said indigenous people in the Standing Rock region were accustomed to mistreatment by police.

“Anybody who knows the history of law enforcement actions in the Dakotas, this is not unusual. But the magnitude of it is extreme,” said Morris, who is the uncle of Red Fawn Fallis, an indigenous protester who was arrested and accused of attempted murder against law enforcement at Standing Rock – a charge her family and supporters strongly deny.

Studies have also found that police kill Native Americans at higher rates than any other racial group, and that indigenous children make up 70% of youth admitted to the federal bureau of prisons despite representing only 1% of the population.

“When someone from Standing Rock goes to Bismarck, we’ll always say, ‘Watch out for the cops,’” said Black Elk, who has lived on the reservation for much of her life.

“The thing is, racism is nothing new here. It has always been there; it’s just bubbling under the surface.”

most viewed

\ No newline at end of file diff --git a/src/extractors/custom/index.js b/src/extractors/custom/index.js index be39bffb3..a6397a579 100644 --- a/src/extractors/custom/index.js +++ b/src/extractors/custom/index.js @@ -23,3 +23,4 @@ export * from './newrepublic.com'; export * from './money.cnn.com'; export * from './www.theverge.com'; export * from './www.cnn.com'; +export * from './www.aol.com'; diff --git a/src/extractors/custom/www.aol.com/index.js b/src/extractors/custom/www.aol.com/index.js new file mode 100644 index 000000000..fa80199c5 --- /dev/null +++ b/src/extractors/custom/www.aol.com/index.js @@ -0,0 +1,53 @@ +export const WwwAolComExtractor = { + domain: 'www.aol.com', + + title: { + selectors: [ + 'h1.p-article__title', + ], + }, + + author: { + selectors: [ + ['meta[name="author"]', 'value'], + ], + }, + + date_published: { + selectors: [ + '.p-article__byline__date', + ], + + timezone: 'America/New_York', + }, + + dek: { + selectors: [ + // enter selectors + ], + }, + + lead_image_url: { + selectors: [ + ['meta[name="og:image"]', 'value'], + ], + }, + + content: { + selectors: [ + '.article-content', + ], + + // Is there anything in the content you selected that needs transformed + // before it's consumable content? E.g., unusual lazy loaded images + transforms: { + }, + + // Is there anything that is in the result that shouldn't be? + // The clean selectors will remove anything that matches from + // the result + clean: [ + + ], + }, +}; diff --git a/src/extractors/custom/www.aol.com/index.test.js b/src/extractors/custom/www.aol.com/index.test.js new file mode 100644 index 000000000..9b435458a --- /dev/null +++ b/src/extractors/custom/www.aol.com/index.test.js @@ -0,0 +1,106 @@ +import assert from 'assert'; +import fs from 'fs'; +import URL from 'url'; +import cheerio from 'cheerio'; + +import Mercury from 'mercury'; +import getExtractor from 'extractors/get-extractor'; +import { excerptContent } from 'utils/text'; + +describe('WwwAolComExtractor', () => { + it('is selected properly', () => { + // This test should be passing by default. + // It sanity checks that the correct parser + // is being selected for URLs from this domain + const url = + 'http://www.aol.com/article/news/2016/12/01/son-of-slain-police-officer-given-teddy-bears-made-from-dads-un/21618553/'; + const extractor = getExtractor(url); + assert.equal(extractor.domain, URL.parse(url).hostname); + }); + + it('returns the title', async () => { + // To pass this test, fill out the title selector + // in ./src/extractors/custom/www.aol.com/index.js. + const html = + fs.readFileSync('./fixtures/www.aol.com/1480618816916.html'); + const articleUrl = + 'http://www.aol.com/article/news/2016/12/01/son-of-slain-police-officer-given-teddy-bears-made-from-dads-un/21618553/'; + + const { title } = + await Mercury.parse(articleUrl, html, { fallback: false }); + + // Update these values with the expected values from + // the article. + assert.equal(title, 'Son of slain police officer given teddy bears made from dad\'s uniform'); + }); + + it('returns the author', async () => { + // To pass this test, fill out the author selector + // in ./src/extractors/custom/www.aol.com/index.js. + const html = + fs.readFileSync('./fixtures/www.aol.com/1480618816916.html'); + const articleUrl = + 'http://www.aol.com/article/news/2016/12/01/son-of-slain-police-officer-given-teddy-bears-made-from-dads-un/21618553/'; + + const { author } = + await Mercury.parse(articleUrl, html, { fallback: false }); + + // Update these values with the expected values from + // the article. + assert.equal(author, 'AOL Staff'); + }); + + it('returns the date_published', async () => { + // To pass this test, fill out the date_published selector + // in ./src/extractors/custom/www.aol.com/index.js. + const html = + fs.readFileSync('./fixtures/www.aol.com/1480618816916.html'); + const articleUrl = + 'http://www.aol.com/article/news/2016/12/01/son-of-slain-police-officer-given-teddy-bears-made-from-dads-un/21618553/'; + + const { date_published } = + await Mercury.parse(articleUrl, html, { fallback: false }); + + // Update these values with the expected values from + // the article. + assert.equal(date_published, '2016-12-01T18:01:00.000Z'); + }); + + it('returns the lead_image_url', async () => { + // To pass this test, fill out the lead_image_url selector + // in ./src/extractors/custom/www.aol.com/index.js. + const html = + fs.readFileSync('./fixtures/www.aol.com/1480618816916.html'); + const articleUrl = + 'http://www.aol.com/article/news/2016/12/01/son-of-slain-police-officer-given-teddy-bears-made-from-dads-un/21618553/'; + + const { lead_image_url } = + await Mercury.parse(articleUrl, html, { fallback: false }); + + // Update these values with the expected values from + // the article. + assert.equal(lead_image_url, 'http://o.aolcdn.com/dims-shared/dims3/GLOB/crop/475x312+0+0/resize/1028x675!/format/jpg/quality/85/http%3A%2F%2Fo.aolcdn.com%2Fhss%2Fstorage%2Fmidas%2Fc8242ab14e089c284b031379d025d64%2F204656928%2FScreen%2BShot%2B2016-12-01%2Bat%2B1.15.51%2BPM.png'); + }); + + it('returns the content', async () => { + // To pass this test, fill out the content selector + // in ./src/extractors/custom/www.aol.com/index.js. + // You may also want to make use of the clean and transform + // options. + const html = + fs.readFileSync('./fixtures/www.aol.com/1480618816916.html'); + const url = + 'http://www.aol.com/article/news/2016/12/01/son-of-slain-police-officer-given-teddy-bears-made-from-dads-un/21618553/'; + + const { content } = + await Mercury.parse(url, html, { fallback: false }); + + const $ = cheerio.load(content || ''); + + const first13 = excerptContent($('*').first().text(), 13); + + // Update these values with the expected values from + // the article. + assert.equal(first13, 'ST. LOUIS, MO (KTVI) – Amid unimaginable grief, the widow of slain Saint'); + }); +});