blob: fe19ab792914f4b3585c190b222f5933830f83c2 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
use strict;
use warnings;
use English;
use Regexp::Common qw(URI);
use LWP::UserAgent;
use HTML::TreeBuilder 5 -weak;
use HTML::HeadParser;
use feature qw(say);
# How to read each file in a directory $dir
my $numargs = $#ARGV + 1;
sub usage {
say "Pass a search term. All lines in the journal will be matched and URLs quoted will be extracted.";
exit;
}
if ($numargs != 1) {
usage();
}
my @targetlines;
my $searchterm = $ARGV[0];
my @urls;
my $dir = '/home/lemon/Notes/journal';
foreach my $f (glob("$dir/*.md")) {
# printf "%s\n", $f;
open my $fh, "<", $f or die "Cannot open that file '$f': $OS_ERROR";
while (<$fh>) {
if ($_ =~ m/$searchterm/) {
# printf " %s", $_;
push @targetlines, $_;
}
}
close $fh or die "can't read close file '$f': $OS_ERROR";
}
sub striptime {
my $url = shift;
$url =~ s/\?t=\d*//;
return $url;
}
#
# Let's interact with the World Wide Web!
my $ua = LWP::UserAgent->new;
$ua->agent("Mozilla/8.0");
foreach my $line (@targetlines) {
# if ($line =~ /(http.*$)/) {
if ($line =~ m/$RE{URI}{HTTP}{-scheme => qr<https?>}{-keep}/) {
my$t = $1;
$t =~ s/\.$//; # remove the fullstop if it has one at the end
push @urls => striptime($t)
# my $req = HTTP::Request->new(GET => $t);
# $req->header(Accept => "text/html");
# my $res = $ua->request($req);
# my $p = HTML::HeadParser->new;
# $p->parse($res->content) and print "not finished";
# print $p->header('Title'), "\n";
# my $root = HTML::TreeBuilder->new_from_content($res->content);
# my $title = $root->look_down('_tag' => 'title');
# my $value = $title->attr('value');
}
}
foreach my $url (@urls) {
print "URL: $url\n";
my $req = HTTP::Request->new(GET => $url);
$req->header(Accept => "text/html");
my $res = $ua->request($req);
my $p = HTML::HeadParser->new;
$p->parse($res->content) and print "not finished";
print "TITLE:", $p->header('Title'), "\n";
print "\n";
}
# foreach my $url (@urls) {
# print $url;
# my $req = HTTP::Request->new(GET => $url);
# $req->header(Accept => "text/html");
# my $res = $ua->request($req);
# my $root = HTML::TreeBuilder->new_from_content($req->content);
# print $root;
# # my @elements = $root->look_down(_tag => "title");
# # foreach my $thing (@elements) {
# # print $thing->as_text, "\n";
# # }
# }
# if ($res->is_success) {
# $tree->parse($res->as_string);
# }
# else {
# print $res->status_line, "\n";
# }
|
