mirror of
https://github.com/RSS-Bridge/rss-bridge.git
synced 2025-04-04 16:49:35 +00:00
Compare commits
335 Commits
2024-02-02
...
master
Author | SHA1 | Date | |
---|---|---|---|
|
d6a9da1cc8 | ||
|
85962e18d3 | ||
|
a19b63e840 | ||
|
5365b57638 | ||
|
462c005f2c | ||
|
db42f2786c | ||
|
26a4c255d3 | ||
|
3055e69c23 | ||
|
7c1e01b45a | ||
|
4d8a46d46e | ||
|
9d6aa5ee38 | ||
|
1c45eff505 | ||
|
68ff39e164 | ||
|
abb1602524 | ||
|
87112497de | ||
|
38bb5115c9 | ||
|
23cb9349fc | ||
|
05a9ac0f06 | ||
|
91fe6c1fae | ||
|
7260f28e10 | ||
|
87ab1e4513 | ||
|
dee734d360 | ||
|
744f996224 | ||
|
f270cd35e7 | ||
|
83c36a87e2 | ||
|
810e17b556 | ||
|
97f07cf216 | ||
|
62fafdc24b | ||
|
cd4cdcfd65 | ||
|
00a24e2f69 | ||
|
92b5e7093f | ||
|
b52f01505d | ||
|
e4c32bb046 | ||
|
dd4dcfa59c | ||
|
4e678c955f | ||
|
549bed64d2 | ||
|
94924d8e16 | ||
|
920b21b1fd | ||
|
935075072b | ||
|
3ae7a10223 | ||
|
bf431a6eae | ||
|
824ac5e373 | ||
|
ae8394d976 | ||
|
4da61b7922 | ||
|
8b1ba003a8 | ||
|
230edf602e | ||
|
bd7d1734c3 | ||
|
dd8bc077ed | ||
|
952a2d99a3 | ||
|
58b3cfb158 | ||
|
028acd0af1 | ||
|
2a58f82bd8 | ||
|
5214581386 | ||
|
eadea242a7 | ||
|
1a2c1f5bba | ||
|
776a1f47f3 | ||
|
39ecd63f72 | ||
|
0e2655fc8a | ||
|
e355276378 | ||
|
cb65125dbd | ||
|
1d02214e12 | ||
|
48cb7d71ed | ||
|
f9e9c8101e | ||
|
97f7df0d06 | ||
|
db3899f2e6 | ||
|
d36cd0a332 | ||
|
662e0bfa95 | ||
|
3fc38c15a3 | ||
|
be51ba17df | ||
|
c44a76ff17 | ||
|
7c6d4a932c | ||
|
45ee018a6e | ||
|
e825272987 | ||
|
97eebfb562 | ||
|
2a44a006b2 | ||
|
974f00cd6a | ||
|
4b4d622333 | ||
|
b4a63e7040 | ||
|
7d544f1fab | ||
|
152e96d3d0 | ||
|
f0db6a22d1 | ||
|
8234906127 | ||
|
d2370320e9 | ||
|
9126b0f982 | ||
|
4685bbdffd | ||
|
bf4a918e60 | ||
|
17d142c038 | ||
|
59d77d4576 | ||
|
d956471d42 | ||
|
6a81fc0f51 | ||
|
88ccc6067c | ||
|
c7f9870ba7 | ||
|
c651e11b0f | ||
|
b42a993176 | ||
|
ec6f98e3c2 | ||
|
74496e23aa | ||
|
83bc3fd762 | ||
|
628b30208a | ||
|
e3260ff529 | ||
|
086ef7f8a7 | ||
|
2ee615e588 | ||
|
a6e8760726 | ||
|
9457e075f6 | ||
|
2294dac3f1 | ||
|
6c86e2c1f7 | ||
|
dd165ea9d1 | ||
|
1cd5b072f3 | ||
|
8d6d0fa10c | ||
|
bd0fb1da99 | ||
|
29d984cbe7 | ||
|
082542dabc | ||
|
bc536f3928 | ||
|
c3dc46a307 | ||
|
6c88f2c21e | ||
|
668f3a9d7e | ||
|
b9eb3c887a | ||
|
f9a51b6768 | ||
|
51cdb66f9c | ||
|
bd88bc27d3 | ||
|
56994b3b5c | ||
|
664436c5f4 | ||
|
70cf917f09 | ||
|
776e27218a | ||
|
e5e2059ed7 | ||
|
e7d6f89887 | ||
|
0c96a47e8c | ||
|
5d83050673 | ||
|
bd823100cd | ||
|
f89c75b4b8 | ||
|
cdf21d48e5 | ||
|
3a5de759fa | ||
|
eb21e97d01 | ||
|
6aba9fdf54 | ||
|
63c16e470d | ||
|
af26d845d9 | ||
|
80c43f10d8 | ||
|
d9316cdc60 | ||
|
40041dd65f | ||
|
358bebbb89 | ||
|
293d04f296 | ||
|
3dc8b65a0b | ||
|
486191b419 | ||
|
a6bdc322b0 | ||
|
36fd72c87e | ||
|
9cabf60144 | ||
|
6a24e53d6c | ||
|
bb2f471a03 | ||
|
3e1a8b29d9 | ||
|
9f48370eb0 | ||
|
39952c2d95 | ||
|
e7ae06dcf0 | ||
|
58544cd61a | ||
|
e010fd4d52 | ||
|
d51cc8f1a7 | ||
|
6516e31c1b | ||
|
c849576c93 | ||
|
b0674d7b19 | ||
|
05e2c350b7 | ||
|
4a3919c1a3 | ||
|
06a8896000 | ||
|
d379f3e575 | ||
|
3a327503ee | ||
|
2d5d2f5017 | ||
|
320afc3f32 | ||
|
c0e37bcf35 | ||
|
e9d3a657ba | ||
|
307c22204d | ||
|
4424ea54e9 | ||
|
133dbf87c5 | ||
|
2e6e246759 | ||
|
129b8a3a5a | ||
|
4ef5ca50c6 | ||
|
adcc8e371d | ||
|
f358f1abec | ||
|
2acd415475 | ||
|
6afd13eb06 | ||
|
2a96bf19b5 | ||
|
9973f731df | ||
|
7073bb2f46 | ||
|
db85015daa | ||
|
8c4385e61d | ||
|
829d570f8e | ||
|
b25a779d98 | ||
|
ee54cf4576 | ||
|
9215b95779 | ||
|
c11bc184ca | ||
|
313be4c512 | ||
|
4faaa79101 | ||
|
6ec9193546 | ||
|
401cc187b7 | ||
|
0051e0fcdd | ||
|
d050fe9a9b | ||
|
8ae716e75c | ||
|
b505667168 | ||
|
615c533587 | ||
|
8a1f2604aa | ||
|
b8a9f34527 | ||
|
e55e9b8fac | ||
|
9982bfce1f | ||
|
1a8d0fb8ab | ||
|
891c8979a3 | ||
|
aa3989873c | ||
|
cb91afbd71 | ||
|
22b39e3fcd | ||
|
6d81d6d306 | ||
|
955fb6f315 | ||
|
8dd56bca05 | ||
|
f773878459 | ||
|
d28a0fd94b | ||
|
bba225dfe8 | ||
|
a1b3e596fc | ||
|
2fcba49433 | ||
|
049af3cef7 | ||
|
376e711f03 | ||
|
00d5242871 | ||
|
f7ddbcd733 | ||
|
da8cfdf179 | ||
|
4539eb69aa | ||
|
8bf1537054 | ||
|
d0c35146dd | ||
|
adad9d6405 | ||
|
2a84350cb2 | ||
|
d60f0b0e74 | ||
|
00074b9bfc | ||
|
206bebc7bd | ||
|
0eac7a0784 | ||
|
649dfa7292 | ||
|
bb1e308057 | ||
|
e1b74aeb1b | ||
|
d3d33c72bd | ||
|
87fa6ea71e | ||
|
36706a3dec | ||
|
cfd406861e | ||
|
bd90109c70 | ||
|
5a68ee0c87 | ||
|
dc199ebf5c | ||
|
75f35391fa | ||
|
7bde7a56f9 | ||
|
4d12aa2a9e | ||
|
a7ed3d56f9 | ||
|
b785a4b64e | ||
|
6e2aeda61d | ||
|
4949900863 | ||
|
776ee233bd | ||
|
1c3024fca7 | ||
|
d11b7f7754 | ||
|
f480209825 | ||
|
d15960f955 | ||
|
f3ca567159 | ||
|
d31f20758c | ||
|
154b8b9cdb | ||
|
1f71d76ac1 | ||
|
8c3e973b9f | ||
|
97f5dafbc5 | ||
|
957a820931 | ||
|
b4d397ff70 | ||
|
89013faf7d | ||
|
428c6c3c66 | ||
|
58c254ad3b | ||
|
a73b66f4d6 | ||
|
815dc180cc | ||
|
7d6881732d | ||
|
4602f4f475 | ||
|
b3ac1d176c | ||
|
d5aa3aef69 | ||
|
3ff2ef94e0 | ||
|
001dd47439 | ||
|
3cba984d22 | ||
|
82606a479a | ||
|
94292af51b | ||
|
f736da6fae | ||
|
fb66775ece | ||
|
8f962383c2 | ||
|
bb979e9e08 | ||
|
a12bab9eed | ||
|
b4659786cb | ||
|
7001fbaf49 | ||
|
d5d470cbc2 | ||
|
182567e434 | ||
|
9682f74fc5 | ||
|
17a3b4c9d8 | ||
|
73289324bd | ||
|
8ca1b90840 | ||
|
1c3c85d8ff | ||
|
d23fd2522c | ||
|
b58d8b099b | ||
|
545dc969d3 | ||
|
24e429969f | ||
|
e0be366258 | ||
|
be445759b6 | ||
|
db984d8a8b | ||
|
e251e358ff | ||
|
0c2099a852 | ||
|
fee5e269d0 | ||
|
2aace6c898 | ||
|
3ed193eee2 | ||
|
58e2b56d40 | ||
|
a61524bf77 | ||
|
36147a082d | ||
|
e6cb5fdc89 | ||
|
4bad1c140a | ||
|
5b80af978f | ||
|
ecf61f6fa7 | ||
|
254efc2812 | ||
|
84b93e0f8f | ||
|
79699131e8 | ||
|
f7c1b71939 | ||
|
7a7f8d5050 | ||
|
683c968d64 | ||
|
4c355ba308 | ||
|
35f6e62e45 | ||
|
932f20d434 | ||
|
e65155f440 | ||
|
7813f4564e | ||
|
4d15ffd2cf | ||
|
598ee5b51e | ||
|
257799be8e | ||
|
8e8028b786 | ||
|
ff7840d60f | ||
|
df7b91a2a3 | ||
|
7b2ac36264 | ||
|
46ac77590e | ||
|
6f731b20a9 | ||
|
8a6798a227 | ||
|
ae2eb2f1d1 | ||
|
cfef482366 | ||
|
75a0a779c0 | ||
|
6bb04d48ed | ||
|
6878eb26aa | ||
|
64f95b4990 | ||
|
66a6847fd0 | ||
|
7931f37a83 | ||
|
d175bab58e | ||
|
7c89712837 | ||
|
a14508d79b |
2
.gitattributes
vendored
2
.gitattributes
vendored
@ -47,8 +47,6 @@ phpcs.xml export-ignore
|
||||
phpcompatibility.xml export-ignore
|
||||
tests/ export-ignore
|
||||
cache/.gitkeep export-ignore
|
||||
bridges/DemoBridge.php export-ignore
|
||||
bridges/FeedExpanderExampleBridge.php export-ignore
|
||||
|
||||
## Composer
|
||||
#
|
||||
|
1
.github/.gitignore
vendored
1
.github/.gitignore
vendored
@ -4,3 +4,4 @@
|
||||
# Generated files
|
||||
comment*.md
|
||||
comment*.txt
|
||||
*.html
|
||||
|
4
.github/ISSUE_TEMPLATE/bridge-request.md
vendored
4
.github/ISSUE_TEMPLATE/bridge-request.md
vendored
@ -49,9 +49,9 @@ Please describe what you expect from the bridge. Whenever possible provide sampl
|
||||
- _Default limit_: 5
|
||||
- [ ] Load full articles
|
||||
- _Cache articles_ (articles are stored in a local cache on first request): yes
|
||||
- _Cache timeout_ (max = 24 hours): 24 hours
|
||||
- _Cache timeout_ : 24 hours
|
||||
- [X] Balance requests (RSS-Bridge uses cached versions to reduce bandwith usage)
|
||||
- _Timeout_ (default = 5 minutes, max = 24 hours): 5 minutes
|
||||
- _Timeout_ (default = 5 minutes): 5 minutes
|
||||
|
||||
<!--Be aware that some options might not be available for your specific request due to technical limitations!-->
|
||||
|
||||
|
86
.github/prtester.py
vendored
86
.github/prtester.py
vendored
@ -4,7 +4,9 @@ import re
|
||||
from bs4 import BeautifulSoup
|
||||
from datetime import datetime
|
||||
from typing import Iterable
|
||||
import os.path
|
||||
import os
|
||||
import glob
|
||||
import urllib
|
||||
|
||||
# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
|
||||
#
|
||||
@ -13,18 +15,33 @@ import os.path
|
||||
# It also add a <base> tag with the url of em's public instance, so viewing
|
||||
# the HTML file locally will actually work as designed.
|
||||
|
||||
ARTIFACT_FILE_EXTENSION = '.html'
|
||||
|
||||
class Instance:
|
||||
name = ''
|
||||
url = ''
|
||||
|
||||
def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: bool, title: str, output_file: str):
|
||||
start_date = datetime.now()
|
||||
|
||||
prid = os.getenv('PR')
|
||||
artifact_base_url = f'https://rss-bridge.github.io/rss-bridge-tests/prs/{prid}'
|
||||
artifact_directory = os.getcwd()
|
||||
for file in glob.glob(f'*{ARTIFACT_FILE_EXTENSION}', root_dir=artifact_directory):
|
||||
os.remove(file)
|
||||
|
||||
table_rows = []
|
||||
for instance in instances:
|
||||
page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page
|
||||
soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
|
||||
bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page
|
||||
table_rows += testBridges(instance, bridge_cards, with_upload, with_reduced_upload) # run the main scraping code with the list of bridges
|
||||
table_rows += testBridges(
|
||||
instance=instance,
|
||||
bridge_cards=bridge_cards,
|
||||
with_upload=with_upload,
|
||||
with_reduced_upload=with_reduced_upload,
|
||||
artifact_directory=artifact_directory,
|
||||
artifact_base_url=artifact_base_url) # run the main scraping code with the list of bridges
|
||||
with open(file=output_file, mode='w+', encoding='utf-8') as file:
|
||||
table_rows_value = '\n'.join(sorted(table_rows))
|
||||
file.write(f'''
|
||||
@ -36,7 +53,7 @@ def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload:
|
||||
*last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}*
|
||||
'''.strip())
|
||||
|
||||
def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool) -> Iterable:
|
||||
def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool, artifact_directory: str, artifact_base_url: str) -> Iterable:
|
||||
instance_suffix = ''
|
||||
if instance.name:
|
||||
instance_suffix = f' ({instance.name})'
|
||||
@ -45,15 +62,14 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w
|
||||
bridgeid = bridge_card.get('id')
|
||||
bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
|
||||
print(f'{bridgeid}{instance_suffix}')
|
||||
bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html'
|
||||
bridge_name = bridgeid.replace('Bridge', '')
|
||||
context_forms = bridge_card.find_all("form")
|
||||
form_number = 1
|
||||
for context_form in context_forms:
|
||||
# a bridge can have multiple contexts, named 'forms' in html
|
||||
# this code will produce a fully working formstring that should create a working feed when called
|
||||
# this code will produce a fully working url that should create a working feed when called
|
||||
# this will create an example feed for every single context, to test them all
|
||||
formstring = ''
|
||||
context_parameters = {}
|
||||
error_messages = []
|
||||
context_name = '*untitled*'
|
||||
context_name_element = context_form.find_previous_sibling('h5')
|
||||
@ -62,27 +78,27 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w
|
||||
parameters = context_form.find_all("input")
|
||||
lists = context_form.find_all("select")
|
||||
# this for/if mess cycles through all available input parameters, checks if it required, then pulls
|
||||
# the default or examplevalue and then combines it all together into the formstring
|
||||
# the default or examplevalue and then combines it all together into the url parameters
|
||||
# if an example or default value is missing for a required attribute, it will throw an error
|
||||
# any non-required fields are not tested!!!
|
||||
for parameter in parameters:
|
||||
if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
|
||||
cleanvalue = parameter.get('value').replace(" ","+")
|
||||
formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue
|
||||
if parameter.get('type') == 'number' or parameter.get('type') == 'text':
|
||||
parameter_type = parameter.get('type')
|
||||
parameter_name = parameter.get('name')
|
||||
if parameter_type == 'hidden':
|
||||
context_parameters[parameter_name] = parameter.get('value')
|
||||
if parameter_type == 'number' or parameter_type == 'text':
|
||||
if parameter.has_attr('required'):
|
||||
if parameter.get('placeholder') == '':
|
||||
if parameter.get('value') == '':
|
||||
name_value = parameter.get('name')
|
||||
error_messages.append(f'Missing example or default value for parameter "{name_value}"')
|
||||
error_messages.append(f'Missing example or default value for parameter "{parameter_name}"')
|
||||
else:
|
||||
formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value')
|
||||
context_parameters[parameter_name] = parameter.get('value')
|
||||
else:
|
||||
formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder')
|
||||
# same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
|
||||
if parameter.get('type') == 'checkbox':
|
||||
context_parameters[parameter_name] = parameter.get('placeholder')
|
||||
# same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the url parameters
|
||||
if parameter_type == 'checkbox':
|
||||
if parameter.has_attr('checked'):
|
||||
formstring = formstring + '&' + parameter.get('name') + '=on'
|
||||
context_parameters[parameter_name] = 'on'
|
||||
for listing in lists:
|
||||
selectionvalue = ''
|
||||
listname = listing.get('name')
|
||||
@ -102,15 +118,21 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w
|
||||
if 'selected' in selectionentry.attrs:
|
||||
selectionvalue = selectionentry.get('value')
|
||||
break
|
||||
formstring = formstring + '&' + listname + '=' + selectionvalue
|
||||
termpad_url = 'about:blank'
|
||||
context_parameters[listname] = selectionvalue
|
||||
artifact_url = 'about:blank'
|
||||
if error_messages:
|
||||
status = '<br>'.join(map(lambda m: f'❌ `{m}`', error_messages))
|
||||
else:
|
||||
# if all example/default values are present, form the full request string, run the request, add a <base> tag with
|
||||
# if all example/default values are present, form the full request url, run the request, add a <base> tag with
|
||||
# the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and
|
||||
# then upload it to termpad.com, a pastebin-like-site.
|
||||
response = requests.get(instance.url + bridgestring + formstring)
|
||||
# then save it to a html file.
|
||||
context_parameters.update({
|
||||
'action': 'display',
|
||||
'bridge': bridgeid,
|
||||
'format': 'Html',
|
||||
})
|
||||
request_url = f'{instance.url}/?{urllib.parse.urlencode(context_parameters)}'
|
||||
response = requests.get(request_url)
|
||||
page_text = response.text.replace('<head>','<head><base href="https://rss-bridge.org/bridge01/" target="_blank">')
|
||||
page_text = page_text.encode("utf_8")
|
||||
soup = BeautifulSoup(page_text, "html.parser")
|
||||
@ -134,16 +156,18 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w
|
||||
if status_is_ok:
|
||||
status = '✔️'
|
||||
if with_upload and (not with_reduced_upload or not status_is_ok):
|
||||
termpad = requests.post(url="https://termpad.com/", data=page_text)
|
||||
termpad_url = termpad.text.strip()
|
||||
termpad_url = termpad_url.replace('termpad.com/','termpad.com/raw/')
|
||||
table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({termpad_url}) | {status} |')
|
||||
filename = f'{bridge_name} {form_number}{instance_suffix}{ARTIFACT_FILE_EXTENSION}'
|
||||
filename = re.sub(r'[^a-z0-9 \_\-\.]', '', filename, flags=re.I).replace(' ', '_')
|
||||
with open(file=f'{artifact_directory}/{filename}', mode='wb') as file:
|
||||
file.write(page_text)
|
||||
artifact_url = f'{artifact_base_url}/{filename}'
|
||||
table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({artifact_url}) | {status} |')
|
||||
form_number += 1
|
||||
return table_rows
|
||||
|
||||
def getFirstLine(value: str) -> str:
|
||||
# trim whitespace and remove text that can break the table or is simply unnecessary
|
||||
clean_value = re.sub('^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip())
|
||||
clean_value = re.sub(r'^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip())
|
||||
first_line = next(iter(clean_value.splitlines()), '')
|
||||
max_length = 250
|
||||
if (len(first_line) > max_length):
|
||||
@ -163,8 +187,8 @@ if __name__ == '__main__':
|
||||
for instance_arg in args.instances:
|
||||
instance_arg_parts = instance_arg.split('::')
|
||||
instance = Instance()
|
||||
instance.name = instance_arg_parts[1] if len(instance_arg_parts) >= 2 else ''
|
||||
instance.url = instance_arg_parts[0]
|
||||
instance.name = instance_arg_parts[1].strip() if len(instance_arg_parts) >= 2 else ''
|
||||
instance.url = instance_arg_parts[0].strip().rstrip("/")
|
||||
instances.append(instance)
|
||||
else:
|
||||
instance = Instance()
|
||||
@ -181,4 +205,4 @@ if __name__ == '__main__':
|
||||
with_reduced_upload=args.reduced_upload and not args.no_upload,
|
||||
title=args.title,
|
||||
output_file=args.output_file
|
||||
);
|
||||
);
|
||||
|
12
.github/workflows/dockerbuild.yml
vendored
12
.github/workflows/dockerbuild.yml
vendored
@ -21,7 +21,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v4
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.DOCKERHUB_SLUG }}
|
||||
@ -33,26 +33,26 @@ jobs:
|
||||
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/20') }}
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2
|
||||
uses: docker/setup-qemu-action@v3
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Login to DockerHub
|
||||
uses: docker/login-action@v2
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v2
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
-
|
||||
name: Build and push
|
||||
uses: docker/bake-action@v2
|
||||
uses: docker/bake-action@v5
|
||||
with:
|
||||
files: |
|
||||
./docker-bake.hcl
|
||||
|
2
.github/workflows/documentation.yml
vendored
2
.github/workflows/documentation.yml
vendored
@ -9,7 +9,7 @@ jobs:
|
||||
documentation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup PHP
|
||||
|
6
.github/workflows/lint.yml
vendored
6
.github/workflows/lint.yml
vendored
@ -13,7 +13,7 @@ jobs:
|
||||
matrix:
|
||||
php-versions: ['7.4']
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php-versions }}
|
||||
@ -26,7 +26,7 @@ jobs:
|
||||
matrix:
|
||||
php-versions: ['7.4']
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php-versions }}
|
||||
@ -38,7 +38,7 @@ jobs:
|
||||
executable_php_files_check:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- run: |
|
||||
if find -name "*.php" -executable -type f -print -exec false {} +
|
||||
then
|
||||
|
67
.github/workflows/prhtmlgenerator.yml
vendored
67
.github/workflows/prhtmlgenerator.yml
vendored
@ -5,15 +5,30 @@ on:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
check-bridges:
|
||||
name: Check if bridges were changed
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
BRIDGES: ${{ steps.check1.outputs.BRIDGES }}
|
||||
steps:
|
||||
- name: Check number of bridges
|
||||
id: check1
|
||||
run: |
|
||||
PR=${{github.event.number}};
|
||||
wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
|
||||
bridgeamount=$(cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq | wc -l);
|
||||
echo "BRIDGES=$bridgeamount" >> "$GITHUB_OUTPUT"
|
||||
test-pr:
|
||||
name: Generate HTML
|
||||
runs-on: ubuntu-latest
|
||||
needs: check-bridges
|
||||
if: needs.check-bridges.outputs.BRIDGES > 0
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
# Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989
|
||||
steps:
|
||||
- name: Check out self
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{github.event.pull_request.head.ref}}
|
||||
repository: ${{github.event.pull_request.head.repo.full_name}}
|
||||
@ -33,9 +48,9 @@ jobs:
|
||||
docker build -t prbuild .;
|
||||
docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3001:80 prbuild
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.7'
|
||||
python-version: '3.13'
|
||||
cache: 'pip'
|
||||
- name: Install requirements
|
||||
run: |
|
||||
@ -51,9 +66,17 @@ jobs:
|
||||
body="${body//$'\n'/'%0A'}";
|
||||
body="${body//$'\r'/'%0D'}";
|
||||
echo "bodylength=${#body}" >> $GITHUB_OUTPUT
|
||||
env:
|
||||
PR: ${{ github.event.number }}
|
||||
- name: Upload generated tests
|
||||
uses: actions/upload-artifact@v4
|
||||
id: upload-generated-tests
|
||||
with:
|
||||
name: tests
|
||||
path: '*.html'
|
||||
- name: Find Comment
|
||||
if: ${{ steps.testrun.outputs.bodylength > 130 }}
|
||||
uses: peter-evans/find-comment@v2
|
||||
uses: peter-evans/find-comment@v3
|
||||
id: fc
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
@ -61,9 +84,43 @@ jobs:
|
||||
body-includes: Pull request artifacts
|
||||
- name: Create or update comment
|
||||
if: ${{ steps.testrun.outputs.bodylength > 130 }}
|
||||
uses: peter-evans/create-or-update-comment@v2
|
||||
uses: peter-evans/create-or-update-comment@v4
|
||||
with:
|
||||
comment-id: ${{ steps.fc.outputs.comment-id }}
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
body-file: comment.txt
|
||||
edit-mode: replace
|
||||
upload_tests:
|
||||
name: Upload tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: test-pr
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: 'RSS-Bridge/rss-bridge-tests'
|
||||
ref: 'main'
|
||||
token: ${{ secrets.RSSTESTER_ACTION }}
|
||||
|
||||
- name: Setup git config
|
||||
run: |
|
||||
git config --global user.name "GitHub Actions"
|
||||
git config --global user.email "<>"
|
||||
|
||||
- name: Download tests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: tests
|
||||
|
||||
- name: Move tests
|
||||
run: |
|
||||
cd prs
|
||||
mkdir -p ${{github.event.number}}
|
||||
cd ${{github.event.number}}
|
||||
mv -f $GITHUB_WORKSPACE/*.html .
|
||||
|
||||
- name: Commit and push generated tests
|
||||
run: |
|
||||
export COMMIT_MESSAGE="Added tests for PR ${{github.event.number}}"
|
||||
git add .
|
||||
git commit -m "$COMMIT_MESSAGE"
|
||||
git push
|
||||
|
4
.github/workflows/tests.yml
vendored
4
.github/workflows/tests.yml
vendored
@ -13,9 +13,11 @@ jobs:
|
||||
matrix:
|
||||
php-versions: ['7.4', '8.0', '8.1']
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php-versions }}
|
||||
env:
|
||||
update: true
|
||||
- run: composer install
|
||||
- run: composer test
|
||||
|
@ -15,7 +15,7 @@
|
||||
* [Astalaseven](https://github.com/Astalaseven)
|
||||
* [Astyan-42](https://github.com/Astyan-42)
|
||||
* [austinhuang0131](https://github.com/austinhuang0131)
|
||||
* [AxorPL](https://github.com/AxorPL)
|
||||
* [axor-mst](https://github.com/axor-mst)
|
||||
* [ayacoo](https://github.com/ayacoo)
|
||||
* [az5he6ch](https://github.com/az5he6ch)
|
||||
* [b1nj](https://github.com/b1nj)
|
||||
@ -23,6 +23,7 @@
|
||||
* [Binnette](https://github.com/Binnette)
|
||||
* [BoboTiG](https://github.com/BoboTiG)
|
||||
* [Bockiii](https://github.com/Bockiii)
|
||||
* [brtsos](https://github.com/brtsos)
|
||||
* [captn3m0](https://github.com/captn3m0)
|
||||
* [chemel](https://github.com/chemel)
|
||||
* [Chouchen](https://github.com/Chouchen)
|
||||
@ -144,6 +145,7 @@
|
||||
* [Niehztog](https://github.com/Niehztog)
|
||||
* [NikNikYkt](https://github.com/NikNikYkt)
|
||||
* [Nono-m0le](https://github.com/Nono-m0le)
|
||||
* [NotsoanoNimus](https://github.com/NotsoanoNimus)
|
||||
* [obsiwitch](https://github.com/obsiwitch)
|
||||
* [Ololbu](https://github.com/Ololbu)
|
||||
* [ORelio](https://github.com/ORelio)
|
||||
|
39
Dockerfile
39
Dockerfile
@ -1,5 +1,3 @@
|
||||
FROM lwthiker/curl-impersonate:0.5-ff-slim-buster AS curlimpersonate
|
||||
|
||||
FROM debian:12-slim AS rssbridge
|
||||
|
||||
LABEL description="RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one."
|
||||
@ -7,7 +5,8 @@ LABEL repository="https://github.com/RSS-Bridge/rss-bridge"
|
||||
LABEL website="https://github.com/RSS-Bridge/rss-bridge"
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && \
|
||||
RUN set -xe && \
|
||||
apt-get update && \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
ca-certificates \
|
||||
nginx \
|
||||
@ -24,18 +23,44 @@ RUN apt-get update && \
|
||||
php-xml \
|
||||
php-zip \
|
||||
# php-zlib is enabled by default with PHP 8.2 in Debian 12
|
||||
# for downloading libcurl-impersonate
|
||||
curl \
|
||||
&& \
|
||||
# install curl-impersonate library
|
||||
curlimpersonate_version=0.6.0 && \
|
||||
{ \
|
||||
{ \
|
||||
[ $(arch) = 'aarch64' ] && \
|
||||
archive="libcurl-impersonate-v${curlimpersonate_version}.aarch64-linux-gnu.tar.gz" && \
|
||||
sha512sum="d04b1eabe71f3af06aa1ce99b39a49c5e1d33b636acedcd9fad163bc58156af5c3eb3f75aa706f335515791f7b9c7a6c40ffdfa47430796483ecef929abd905d" \
|
||||
; } \
|
||||
|| { \
|
||||
[ $(arch) = 'armv7l' ] && \
|
||||
archive="libcurl-impersonate-v${curlimpersonate_version}.arm-linux-gnueabihf.tar.gz" && \
|
||||
sha512sum="05906b4efa1a6ed8f3b716fd83d476b6eea6bfc68e3dbc5212d65a2962dcaa7bd1f938c9096a7535252b11d1d08fb93adccc633585ff8cb8cec5e58bfe969bc9" \
|
||||
; } \
|
||||
|| { \
|
||||
[ $(arch) = 'x86_64' ] && \
|
||||
archive="libcurl-impersonate-v${curlimpersonate_version}.x86_64-linux-gnu.tar.gz" && \
|
||||
sha512sum="480bbe9452cd9aff2c0daaaf91f1057b3a96385f79011628a9237223757a9b0d090c59cb5982dc54ea0d07191657299ea91ca170a25ced3d7d410fcdff130ace" \
|
||||
; } \
|
||||
} && \
|
||||
curl -LO "https://github.com/lwthiker/curl-impersonate/releases/download/v${curlimpersonate_version}/${archive}" && \
|
||||
echo "$sha512sum $archive" | sha512sum -c - && \
|
||||
mkdir -p /usr/local/lib/curl-impersonate && \
|
||||
tar xaf "$archive" -C /usr/local/lib/curl-impersonate --wildcards 'libcurl-impersonate-ff.so*' && \
|
||||
rm "$archive" && \
|
||||
apt-get purge --assume-yes curl && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so
|
||||
ENV CURL_IMPERSONATE ff91esr
|
||||
|
||||
# logs should go to stdout / stderr
|
||||
RUN ln -sfT /dev/stderr /var/log/nginx/error.log; \
|
||||
ln -sfT /dev/stdout /var/log/nginx/access.log; \
|
||||
chown -R --no-dereference www-data:adm /var/log/nginx/
|
||||
|
||||
COPY --from=curlimpersonate /usr/local/lib/libcurl-impersonate-ff.so /usr/local/lib/curl-impersonate/
|
||||
ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so
|
||||
ENV CURL_IMPERSONATE ff91esr
|
||||
|
||||
COPY ./config/nginx.conf /etc/nginx/sites-available/default
|
||||
COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf
|
||||
COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.ini
|
||||
|
108
README.md
108
README.md
@ -29,7 +29,7 @@ Requires minimum PHP 7.4.
|
||||
|||
|
||||
|||
|
||||
|
||||
## A subset of bridges (16/447)
|
||||
## A subset of bridges (15/447)
|
||||
|
||||
* `CssSelectorBridge`: [Scrape out a feed using CSS selectors](https://rss-bridge.org/bridge01/#bridge-CssSelectorBridge)
|
||||
* `FeedMergeBridge`: [Combine multiple feeds into one](https://rss-bridge.org/bridge01/#bridge-FeedMergeBridge)
|
||||
@ -44,7 +44,6 @@ Requires minimum PHP 7.4.
|
||||
* `ThePirateBayBridge:` [Fetches torrents by search/user/category](https://rss-bridge.org/bridge01/#bridge-ThePirateBayBridge)
|
||||
* `TikTokBridge`: [Fetches posts by username](https://rss-bridge.org/bridge01/#bridge-TikTokBridge)
|
||||
* `TwitchBridge`: [Fetches videos from channel](https://rss-bridge.org/bridge01/#bridge-TwitchBridge)
|
||||
* `VkBridge`: [Fetches posts from user/group](https://rss-bridge.org/bridge01/#bridge-VkBridge)
|
||||
* `XPathBridge`: [Scrape out a feed using XPath expressions](https://rss-bridge.org/bridge01/#bridge-XPathBridge)
|
||||
* `YoutubeBridge`: [Fetches videos by username/channel/playlist/search](https://rss-bridge.org/bridge01/#bridge-YoutubeBridge)
|
||||
* `YouTubeCommunityTabBridge`: [Fetches posts from a channel's community tab](https://rss-bridge.org/bridge01/#bridge-YouTubeCommunityTabBridge)
|
||||
@ -53,9 +52,10 @@ Requires minimum PHP 7.4.
|
||||
|
||||
### How to install on traditional shared web hosting
|
||||
|
||||
RSS-Bridge can basically be unzipped in a web folder. Should be working instantly.
|
||||
RSS-Bridge can basically be unzipped into a web folder. Should be working instantly.
|
||||
|
||||
Latest zip as of Sep 2023: https://github.com/RSS-Bridge/rss-bridge/archive/refs/tags/2023-09-24.zip
|
||||
Latest zip:
|
||||
https://github.com/RSS-Bridge/rss-bridge/archive/refs/heads/master.zip (2MB)
|
||||
|
||||
### How to install on Debian 12 (nginx + php-fpm)
|
||||
|
||||
@ -64,34 +64,34 @@ These instructions have been tested on a fresh Debian 12 VM from Digital Ocean (
|
||||
```shell
|
||||
timedatectl set-timezone Europe/Oslo
|
||||
|
||||
apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl
|
||||
apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl php-intl
|
||||
|
||||
# Create a new user account
|
||||
# Create a user account
|
||||
useradd --shell /bin/bash --create-home rss-bridge
|
||||
|
||||
cd /var/www
|
||||
|
||||
# Create folder and change ownership
|
||||
# Create folder and change its ownership to rss-bridge
|
||||
mkdir rss-bridge && chown rss-bridge:rss-bridge rss-bridge/
|
||||
|
||||
# Become user
|
||||
# Become rss-bridge
|
||||
su rss-bridge
|
||||
|
||||
# Fetch latest master
|
||||
# Clone master branch into existing folder
|
||||
git clone https://github.com/RSS-Bridge/rss-bridge.git rss-bridge/
|
||||
cd rss-bridge
|
||||
|
||||
# Copy over the default config
|
||||
# Copy over the default config (OPTIONAL)
|
||||
cp -v config.default.ini.php config.ini.php
|
||||
|
||||
# Give full permissions only to owner (rss-bridge)
|
||||
chmod 700 -R ./
|
||||
# Recursively give full permissions to user/owner
|
||||
chmod 700 --recursive ./
|
||||
|
||||
# Give read and execute to others (nginx and php-fpm)
|
||||
# Give read and execute to others on folder ./static
|
||||
chmod o+rx ./ ./static
|
||||
|
||||
# Give read to others (nginx)
|
||||
chmod o+r -R ./static
|
||||
# Recursively give give read to others on folder ./static
|
||||
chmod o+r --recursive ./static
|
||||
```
|
||||
|
||||
Nginx config:
|
||||
@ -101,37 +101,37 @@ Nginx config:
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
|
||||
# TODO: change to your own server name
|
||||
server_name example.com;
|
||||
|
||||
access_log /var/log/nginx/rss-bridge.access.log;
|
||||
error_log /var/log/nginx/rss-bridge.error.log;
|
||||
log_not_found off;
|
||||
|
||||
# Intentionally not setting a root folder here
|
||||
|
||||
# autoindex is off by default but feels good to explicitly turn off
|
||||
autoindex off;
|
||||
# Intentionally not setting a root folder
|
||||
|
||||
# Static content only served here
|
||||
location /static/ {
|
||||
alias /var/www/rss-bridge/static/;
|
||||
}
|
||||
|
||||
# Pass off to php-fpm only when location is exactly /
|
||||
# Pass off to php-fpm only when location is EXACTLY == /
|
||||
location = / {
|
||||
root /var/www/rss-bridge/;
|
||||
include snippets/fastcgi-php.conf;
|
||||
fastcgi_read_timeout 45s;
|
||||
fastcgi_pass unix:/run/php/rss-bridge.sock;
|
||||
}
|
||||
|
||||
# Reduce spam
|
||||
# Reduce log noise
|
||||
location = /favicon.ico {
|
||||
access_log off;
|
||||
log_not_found off;
|
||||
}
|
||||
|
||||
# Reduce spam
|
||||
# Reduce log noise
|
||||
location = /robots.txt {
|
||||
access_log off;
|
||||
log_not_found off;
|
||||
}
|
||||
}
|
||||
```
|
||||
@ -150,8 +150,11 @@ listen = /run/php/rss-bridge.sock
|
||||
listen.owner = www-data
|
||||
listen.group = www-data
|
||||
|
||||
; Create 10 workers standing by to serve requests
|
||||
pm = static
|
||||
pm.max_children = 10
|
||||
|
||||
; Respawn worker after 500 requests (workaround for memory leaks etc.)
|
||||
pm.max_requests = 500
|
||||
```
|
||||
|
||||
@ -167,12 +170,10 @@ Restart fpm and nginx:
|
||||
|
||||
```shell
|
||||
# Lint and restart php-fpm
|
||||
php-fpm8.2 -t
|
||||
systemctl restart php8.2-fpm
|
||||
php-fpm8.2 -t && systemctl restart php8.2-fpm
|
||||
|
||||
# Lint and restart nginx
|
||||
nginx -t
|
||||
systemctl restart nginx
|
||||
nginx -t && systemctl restart nginx
|
||||
```
|
||||
|
||||
### How to install from Composer
|
||||
@ -181,7 +182,7 @@ Install the latest release.
|
||||
|
||||
```shell
|
||||
cd /var/www
|
||||
composer create-project -v --no-dev rss-bridge/rss-bridge
|
||||
composer create-project -v --no-dev --no-scripts rss-bridge/rss-bridge
|
||||
```
|
||||
|
||||
### How to install with Caddy
|
||||
@ -194,8 +195,16 @@ Install by downloading the docker image from Docker Hub:
|
||||
|
||||
```bash
|
||||
# Create container
|
||||
docker create --name=rss-bridge --publish 3000:80 rssbridge/rss-bridge
|
||||
docker create --name=rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rssbridge/rss-bridge
|
||||
```
|
||||
|
||||
You can put custom `config.ini.php` and bridges into `./config`.
|
||||
|
||||
**You must restart container for custom changes to take effect.**
|
||||
|
||||
See `docker-entrypoint.sh` for details.
|
||||
|
||||
```bash
|
||||
# Start container
|
||||
docker start rss-bridge
|
||||
```
|
||||
@ -209,30 +218,29 @@ Browse http://localhost:3000/
|
||||
docker build -t rss-bridge .
|
||||
|
||||
# Create container
|
||||
docker create --name rss-bridge --publish 3000:80 rss-bridge
|
||||
docker create --name rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rss-bridge
|
||||
```
|
||||
|
||||
You can put custom `config.ini.php` and bridges into `./config`.
|
||||
|
||||
**You must restart container for custom changes to take effect.**
|
||||
|
||||
See `docker-entrypoint.sh` for details.
|
||||
|
||||
```bash
|
||||
# Start container
|
||||
docker start rss-bridge
|
||||
```
|
||||
|
||||
Browse http://localhost:3000/
|
||||
|
||||
### Install with docker-compose
|
||||
### Install with docker-compose (using Docker Hub)
|
||||
|
||||
Create a `docker-compose.yml` file locally with with the following content:
|
||||
```yml
|
||||
version: '2'
|
||||
services:
|
||||
rss-bridge:
|
||||
image: rssbridge/rss-bridge:latest
|
||||
volumes:
|
||||
- </local/custom/path>:/config
|
||||
ports:
|
||||
- 3000:80
|
||||
restart: unless-stopped
|
||||
```
|
||||
You can put custom `config.ini.php` and bridges into `./config`.
|
||||
|
||||
Then launch with `docker-compose`:
|
||||
**You must restart container for custom changes to take effect.**
|
||||
|
||||
See `docker-entrypoint.sh` for details.
|
||||
|
||||
```bash
|
||||
docker-compose up
|
||||
@ -420,7 +428,16 @@ See `formats/PlaintextFormat.php` for an example.
|
||||
|
||||
These commands require that you have installed the dev dependencies in `composer.json`.
|
||||
|
||||
Run all tests:
|
||||
|
||||
./vendor/bin/phpunit
|
||||
|
||||
Run a single test class:
|
||||
|
||||
./vendor/bin/phpunit --filter UrlTest
|
||||
|
||||
Run linter:
|
||||
|
||||
./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./
|
||||
|
||||
https://github.com/squizlabs/PHP_CodeSniffer/wiki
|
||||
@ -443,7 +460,6 @@ See [CONTRIBUTORS.md](CONTRIBUTORS.md)
|
||||
|
||||
RSS-Bridge uses caching to prevent services from banning your server for repeatedly updating feeds.
|
||||
The specific cache duration can be different between bridges.
|
||||
Cached files are deleted automatically after 24 hours.
|
||||
|
||||
RSS-Bridge allows you to take full control over which bridges are displayed to the user.
|
||||
That way you can host your own RSS-Bridge service with your favorite collection of bridges!
|
||||
|
@ -14,12 +14,13 @@ class ConnectivityAction implements ActionInterface
|
||||
{
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->bridgeFactory = new BridgeFactory();
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function execute(Request $request)
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
if (!Debug::isEnabled()) {
|
||||
return new Response('This action is only available in debug mode!', 403);
|
||||
@ -27,7 +28,7 @@ class ConnectivityAction implements ActionInterface
|
||||
|
||||
$bridgeName = $request->get('bridge');
|
||||
if (!$bridgeName) {
|
||||
return render_template('connectivity.html.php');
|
||||
return new Response(render_template('connectivity.html.php'));
|
||||
}
|
||||
$bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
|
||||
if (!$bridgeClassName) {
|
||||
@ -54,8 +55,8 @@ class ConnectivityAction implements ActionInterface
|
||||
];
|
||||
try {
|
||||
$response = getContents($bridge::URI, [], $curl_opts, true);
|
||||
$result['http_code'] = $response['code'];
|
||||
if (in_array($response['code'], [200])) {
|
||||
$result['http_code'] = $response->getCode();
|
||||
if (in_array($result['http_code'], [200])) {
|
||||
$result['successful'] = true;
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
|
@ -2,7 +2,15 @@
|
||||
|
||||
class DetectAction implements ActionInterface
|
||||
{
|
||||
public function execute(Request $request)
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$url = $request->get('url');
|
||||
$format = $request->get('format');
|
||||
@ -14,14 +22,12 @@ class DetectAction implements ActionInterface
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']));
|
||||
}
|
||||
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
|
||||
foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
if (!$bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$bridge = $bridgeFactory->create($bridgeClassName);
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$bridgeParams = $bridge->detectParameters($url);
|
||||
|
||||
|
@ -4,42 +4,28 @@ class DisplayAction implements ActionInterface
|
||||
{
|
||||
private CacheInterface $cache;
|
||||
private Logger $logger;
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->cache = RssBridge::getCache();
|
||||
$this->logger = RssBridge::getLogger();
|
||||
public function __construct(
|
||||
CacheInterface $cache,
|
||||
Logger $logger,
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->cache = $cache;
|
||||
$this->logger = $logger;
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function execute(Request $request)
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$bridgeName = $request->get('bridge');
|
||||
$format = $request->get('format');
|
||||
$noproxy = $request->get('_noproxy');
|
||||
|
||||
$cacheKey = 'http_' . json_encode($request->toArray());
|
||||
/** @var Response $cachedResponse */
|
||||
$cachedResponse = $this->cache->get($cacheKey);
|
||||
if ($cachedResponse) {
|
||||
$ifModifiedSince = $_SERVER['HTTP_IF_MODIFIED_SINCE'] ?? null;
|
||||
$lastModified = $cachedResponse->getHeader('last-modified');
|
||||
if ($ifModifiedSince && $lastModified) {
|
||||
$lastModified = new \DateTimeImmutable($lastModified);
|
||||
$lastModifiedTimestamp = $lastModified->getTimestamp();
|
||||
$modifiedSince = strtotime($ifModifiedSince);
|
||||
if ($lastModifiedTimestamp <= $modifiedSince) {
|
||||
$modificationTimeGMT = gmdate('D, d M Y H:i:s ', $lastModifiedTimestamp);
|
||||
return new Response('', 304, ['last-modified' => $modificationTimeGMT . 'GMT']);
|
||||
}
|
||||
}
|
||||
return $cachedResponse;
|
||||
}
|
||||
|
||||
if (!$bridgeName) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge parameter']), 400);
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge name parameter']), 400);
|
||||
}
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
$bridgeClassName = $bridgeFactory->createBridgeClassName($bridgeName);
|
||||
$bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
|
||||
if (!$bridgeClassName) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Bridge not found']), 404);
|
||||
}
|
||||
@ -47,11 +33,11 @@ class DisplayAction implements ActionInterface
|
||||
if (!$format) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']), 400);
|
||||
}
|
||||
if (!$bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400);
|
||||
}
|
||||
|
||||
|
||||
// Disable proxy (if enabled and per user's request)
|
||||
if (
|
||||
Configuration::getConfig('proxy', 'url')
|
||||
&& Configuration::getConfig('proxy', 'by_bridge')
|
||||
@ -61,9 +47,9 @@ class DisplayAction implements ActionInterface
|
||||
define('NOPROXY', true);
|
||||
}
|
||||
|
||||
$bridge = $bridgeFactory->create($bridgeClassName);
|
||||
$formatFactory = new FormatFactory();
|
||||
$format = $formatFactory->create($format);
|
||||
$cacheKey = 'http_' . json_encode($request->toArray());
|
||||
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$response = $this->createResponse($request, $bridge, $format);
|
||||
|
||||
@ -77,26 +63,12 @@ class DisplayAction implements ActionInterface
|
||||
$this->cache->set($cacheKey, $response, $ttl);
|
||||
}
|
||||
|
||||
if (in_array($response->getCode(), [403, 429, 503])) {
|
||||
// Cache these responses for about ~20 mins on average
|
||||
$this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10));
|
||||
}
|
||||
|
||||
if ($response->getCode() === 500) {
|
||||
$this->cache->set($cacheKey, $response, 60 * 15);
|
||||
}
|
||||
|
||||
if (rand(1, 100) === 2) {
|
||||
$this->cache->prune();
|
||||
}
|
||||
|
||||
return $response;
|
||||
}
|
||||
|
||||
private function createResponse(Request $request, BridgeAbstract $bridge, FormatAbstract $format)
|
||||
private function createResponse(Request $request, BridgeAbstract $bridge, string $format)
|
||||
{
|
||||
$items = [];
|
||||
$feed = [];
|
||||
|
||||
try {
|
||||
$bridge->loadConfiguration();
|
||||
@ -116,28 +88,23 @@ class DisplayAction implements ActionInterface
|
||||
$bridge->setInput($input);
|
||||
$bridge->collectData();
|
||||
$items = $bridge->getItems();
|
||||
if (isset($items[0]) && is_array($items[0])) {
|
||||
$feedItems = [];
|
||||
foreach ($items as $item) {
|
||||
$feedItems[] = FeedItem::fromArray($item);
|
||||
}
|
||||
$items = $feedItems;
|
||||
} catch (\Throwable $e) {
|
||||
if ($e instanceof RateLimitException) {
|
||||
// These are internally generated by bridges
|
||||
$this->logger->info(sprintf('RateLimitException in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429);
|
||||
}
|
||||
$feed = $bridge->getFeed();
|
||||
} catch (\Exception $e) {
|
||||
// Probably an exception inside a bridge
|
||||
if ($e instanceof HttpException) {
|
||||
// Reproduce (and log) these responses regardless of error output and report limit
|
||||
if ($e->getCode() === 429) {
|
||||
$this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429);
|
||||
}
|
||||
if ($e->getCode() === 503) {
|
||||
$this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 503);
|
||||
if (in_array($e->getCode(), [429, 503])) {
|
||||
// Log with debug, immediately reproduce and return
|
||||
$this->logger->debug(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), $e->getCode());
|
||||
}
|
||||
// Some other status code which we let fail normally (but don't log it)
|
||||
} else {
|
||||
// Log error if it's not an HttpException
|
||||
$this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]);
|
||||
}
|
||||
$this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]);
|
||||
$errorOutput = Configuration::getConfig('error', 'output');
|
||||
$reportLimit = Configuration::getConfig('error', 'report_limit');
|
||||
$errorCount = 1;
|
||||
@ -148,7 +115,7 @@ class DisplayAction implements ActionInterface
|
||||
if ($errorCount >= $reportLimit) {
|
||||
if ($errorOutput === 'feed') {
|
||||
// Render the exception as a feed item
|
||||
$items[] = $this->createFeedItemFromException($e, $bridge);
|
||||
$items = [$this->createFeedItemFromException($e, $bridge)];
|
||||
} elseif ($errorOutput === 'http') {
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 500);
|
||||
} elseif ($errorOutput === 'none') {
|
||||
@ -157,38 +124,49 @@ class DisplayAction implements ActionInterface
|
||||
}
|
||||
}
|
||||
|
||||
$formatFactory = new FormatFactory();
|
||||
$format = $formatFactory->create($format);
|
||||
|
||||
$format->setItems($items);
|
||||
$format->setFeed($feed);
|
||||
$format->setFeed($bridge->getFeed());
|
||||
$now = time();
|
||||
$format->setLastModified($now);
|
||||
$headers = [
|
||||
'last-modified' => gmdate('D, d M Y H:i:s ', $now) . 'GMT',
|
||||
'content-type' => $format->getMimeType() . '; charset=' . $format->getCharset(),
|
||||
'content-type' => $format->getMimeType() . '; charset=UTF-8',
|
||||
];
|
||||
return new Response($format->stringify(), 200, $headers);
|
||||
$body = $format->render();
|
||||
|
||||
// This is supposed to remove non-utf8 byte sequences, but I'm unsure if it works
|
||||
ini_set('mbstring.substitute_character', 'none');
|
||||
$body = mb_convert_encoding($body, 'UTF-8', 'UTF-8');
|
||||
|
||||
return new Response($body, 200, $headers);
|
||||
}
|
||||
|
||||
private function createFeedItemFromException($e, BridgeAbstract $bridge): FeedItem
|
||||
private function createFeedItemFromException($e, BridgeAbstract $bridge): array
|
||||
{
|
||||
$item = new FeedItem();
|
||||
$item = [];
|
||||
|
||||
// Create a unique identifier every 24 hours
|
||||
$uniqueIdentifier = urlencode((int)(time() / 86400));
|
||||
$title = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier);
|
||||
$item->setTitle($title);
|
||||
$item->setURI(get_current_url());
|
||||
$item->setTimestamp(time());
|
||||
|
||||
$item['title'] = $title;
|
||||
$item['uri'] = get_current_url();
|
||||
$item['timestamp'] = time();
|
||||
|
||||
// Create an item identifier for feed readers e.g. "staysafetv twitch videos_19389"
|
||||
$item->setUid($bridge->getName() . '_' . $uniqueIdentifier);
|
||||
$item['uid'] = $bridge->getName() . '_' . $uniqueIdentifier;
|
||||
|
||||
$content = render_template(__DIR__ . '/../templates/bridge-error.html.php', [
|
||||
'error' => render_template(__DIR__ . '/../templates/exception.html.php', ['e' => $e]),
|
||||
'searchUrl' => self::createGithubSearchUrl($bridge),
|
||||
'issueUrl' => self::createGithubIssueUrl($bridge, $e, create_sane_exception_message($e)),
|
||||
'issueUrl' => self::createGithubIssueUrl($bridge, $e),
|
||||
'maintainer' => $bridge->getMaintainer(),
|
||||
]);
|
||||
$item->setContent($content);
|
||||
$item['content'] = $content;
|
||||
|
||||
return $item;
|
||||
}
|
||||
|
||||
@ -213,22 +191,34 @@ class DisplayAction implements ActionInterface
|
||||
return $report['count'];
|
||||
}
|
||||
|
||||
private static function createGithubIssueUrl($bridge, $e, string $message): string
|
||||
private static function createGithubIssueUrl(BridgeAbstract $bridge, \Throwable $e): string
|
||||
{
|
||||
return sprintf('https://github.com/RSS-Bridge/rss-bridge/issues/new?%s', http_build_query([
|
||||
'title' => sprintf('%s failed with error %s', $bridge->getName(), $e->getCode()),
|
||||
$maintainer = $bridge->getMaintainer();
|
||||
if (str_contains($maintainer, ',')) {
|
||||
$maintainers = explode(',', $maintainer);
|
||||
} else {
|
||||
$maintainers = [$maintainer];
|
||||
}
|
||||
$maintainers = array_map('trim', $maintainers);
|
||||
|
||||
$queryString = $_SERVER['QUERY_STRING'] ?? '';
|
||||
$query = [
|
||||
'title' => $bridge->getName() . ' failed with: ' . $e->getMessage(),
|
||||
'body' => sprintf(
|
||||
"```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```",
|
||||
$message,
|
||||
"```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```\nMaintainer: @%s",
|
||||
create_sane_exception_message($e),
|
||||
implode("\n", trace_to_call_points(trace_from_exception($e))),
|
||||
$_SERVER['QUERY_STRING'] ?? '',
|
||||
$queryString,
|
||||
Configuration::getVersion(),
|
||||
PHP_OS_FAMILY,
|
||||
phpversion() ?: 'Unknown'
|
||||
phpversion() ?: 'Unknown',
|
||||
implode(', @', $maintainers),
|
||||
),
|
||||
'labels' => 'Bridge-Broken',
|
||||
'assignee' => $bridge->getMaintainer(),
|
||||
]));
|
||||
'assignee' => $maintainer[0],
|
||||
];
|
||||
|
||||
return 'https://github.com/RSS-Bridge/rss-bridge/issues/new?' . http_build_query($query);
|
||||
}
|
||||
|
||||
private static function createGithubSearchUrl($bridge): string
|
||||
|
@ -7,7 +7,15 @@
|
||||
*/
|
||||
class FindfeedAction implements ActionInterface
|
||||
{
|
||||
public function execute(Request $request)
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$url = $request->get('url');
|
||||
$format = $request->get('format');
|
||||
@ -19,15 +27,13 @@ class FindfeedAction implements ActionInterface
|
||||
return new Response('You must specify a format', 400);
|
||||
}
|
||||
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
|
||||
$results = [];
|
||||
foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
if (!$bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$bridge = $bridgeFactory->create($bridgeClassName);
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$bridgeParams = $bridge->detectParameters($url);
|
||||
|
||||
|
@ -2,15 +2,24 @@
|
||||
|
||||
final class FrontpageAction implements ActionInterface
|
||||
{
|
||||
public function execute(Request $request)
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$token = $request->getAttribute('token');
|
||||
|
||||
$messages = [];
|
||||
$activeBridges = 0;
|
||||
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
$bridgeClassNames = $bridgeFactory->getBridgeClassNames();
|
||||
$bridgeClassNames = $this->bridgeFactory->getBridgeClassNames();
|
||||
|
||||
foreach ($bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) {
|
||||
foreach ($this->bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) {
|
||||
$messages[] = [
|
||||
'body' => sprintf('Warning : Bridge "%s" not found', $missingEnabledBridge),
|
||||
'level' => 'warning'
|
||||
@ -19,20 +28,22 @@ final class FrontpageAction implements ActionInterface
|
||||
|
||||
$body = '';
|
||||
foreach ($bridgeClassNames as $bridgeClassName) {
|
||||
if ($bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
$body .= BridgeCard::render($bridgeClassName, $request);
|
||||
if ($this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
$body .= BridgeCard::render($this->bridgeFactory, $bridgeClassName, $token);
|
||||
$activeBridges++;
|
||||
}
|
||||
}
|
||||
|
||||
// todo: cache this renderered template?
|
||||
return render(__DIR__ . '/../templates/frontpage.html.php', [
|
||||
'messages' => $messages,
|
||||
'admin_email' => Configuration::getConfig('admin', 'email'),
|
||||
'admin_telegram' => Configuration::getConfig('admin', 'telegram'),
|
||||
'bridges' => $body,
|
||||
'active_bridges' => $activeBridges,
|
||||
'total_bridges' => count($bridgeClassNames),
|
||||
]);
|
||||
$response = new Response(render(__DIR__ . '/../templates/frontpage.html.php', [
|
||||
'messages' => $messages,
|
||||
'admin_email' => Configuration::getConfig('admin', 'email'),
|
||||
'admin_telegram' => Configuration::getConfig('admin', 'telegram'),
|
||||
'bridges' => $body,
|
||||
'active_bridges' => $activeBridges,
|
||||
'total_bridges' => count($bridgeClassNames),
|
||||
]));
|
||||
|
||||
// TODO: The rendered template could be cached, but beware config changes that changes the html
|
||||
return $response;
|
||||
}
|
||||
}
|
||||
|
@ -4,7 +4,7 @@ declare(strict_types=1);
|
||||
|
||||
class HealthAction implements ActionInterface
|
||||
{
|
||||
public function execute(Request $request)
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$response = [
|
||||
'code' => 200,
|
||||
|
@ -2,19 +2,25 @@
|
||||
|
||||
class ListAction implements ActionInterface
|
||||
{
|
||||
public function execute(Request $request)
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$list = new \stdClass();
|
||||
$list->bridges = [];
|
||||
$list->total = 0;
|
||||
|
||||
$bridgeFactory = new BridgeFactory();
|
||||
|
||||
foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
$bridge = $bridgeFactory->create($bridgeClassName);
|
||||
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$list->bridges[$bridgeClassName] = [
|
||||
'status' => $bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive',
|
||||
'status' => $this->bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive',
|
||||
'uri' => $bridge->getURI(),
|
||||
'donationUri' => $bridge->getDonationURI(),
|
||||
'name' => $bridge->getName(),
|
||||
|
@ -6,9 +6,11 @@
|
||||
*/
|
||||
|
||||
require __DIR__ . '/../lib/bootstrap.php';
|
||||
require __DIR__ . '/../lib/config.php';
|
||||
|
||||
$rssBridge = new RssBridge();
|
||||
$container = require __DIR__ . '/../lib/dependencies.php';
|
||||
|
||||
$cache = RssBridge::getCache();
|
||||
/** @var CacheInterface $cache */
|
||||
$cache = $container['cache'];
|
||||
|
||||
$cache->clear();
|
||||
|
@ -6,9 +6,19 @@
|
||||
*/
|
||||
|
||||
require __DIR__ . '/../lib/bootstrap.php';
|
||||
require __DIR__ . '/../lib/config.php';
|
||||
|
||||
$rssBridge = new RssBridge();
|
||||
$container = require __DIR__ . '/../lib/dependencies.php';
|
||||
|
||||
$cache = RssBridge::getCache();
|
||||
if (
|
||||
Configuration::getConfig('cache', 'type') === 'file'
|
||||
&& !Configuration::getConfig('FileCache', 'enable_purge')
|
||||
) {
|
||||
// Override enable_purge for this particular execution
|
||||
Configuration::setConfig('FileCache', 'enable_purge', true);
|
||||
}
|
||||
|
||||
/** @var CacheInterface $cache */
|
||||
$cache = $container['cache'];
|
||||
|
||||
$cache->prune();
|
||||
|
20
bin/test
Executable file
20
bin/test
Executable file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
/**
|
||||
* Add log records to all three levels (for testing purposes)
|
||||
*/
|
||||
|
||||
require __DIR__ . '/../lib/bootstrap.php';
|
||||
require __DIR__ . '/../lib/config.php';
|
||||
|
||||
$container = require __DIR__ . '/../lib/dependencies.php';
|
||||
|
||||
/** @var Logger $logger */
|
||||
$logger = $container['logger'];
|
||||
|
||||
$logger->debug('This is a test debug message');
|
||||
|
||||
$logger->info('This is a test info message');
|
||||
|
||||
$logger->error('This is a test error message');
|
@ -31,17 +31,17 @@ class ABCNewsBridge extends BridgeAbstract
|
||||
{
|
||||
$url = sprintf('https://www.abc.net.au/news/%s', $this->getInput('topic'));
|
||||
$dom = getSimpleHTMLDOM($url);
|
||||
$dom = $dom->find('div[data-component="CardList"]', 0);
|
||||
$dom = $dom->find('div[data-component="PaginationList"]', 0);
|
||||
if (!$dom) {
|
||||
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
|
||||
}
|
||||
$dom = defaultLinkTo($dom, $this->getURI());
|
||||
foreach ($dom->find('div[data-component="GenericCard"]') as $article) {
|
||||
foreach ($dom->find('article[data-component="DetailCard"]') as $article) {
|
||||
$a = $article->find('a', 0);
|
||||
$this->items[] = [
|
||||
'title' => $a->plaintext,
|
||||
'uri' => $a->href,
|
||||
'content' => $article->find('[data-component="CardDescription"]', 0)->plaintext,
|
||||
'content' => $article->find('p', 0)->plaintext,
|
||||
'timestamp' => strtotime($article->find('time', 0)->datetime),
|
||||
];
|
||||
}
|
||||
|
@ -12,9 +12,29 @@ class AO3Bridge extends BridgeAbstract
|
||||
'url' => [
|
||||
'name' => 'url',
|
||||
'required' => true,
|
||||
// Example: F/F tag, complete works only
|
||||
'exampleValue' => 'https://archiveofourown.org/works?work_search[complete]=T&tag_id=F*s*F',
|
||||
// Example: F/F tag
|
||||
'exampleValue' => 'https://archiveofourown.org/tags/F*s*F/works',
|
||||
],
|
||||
'range' => [
|
||||
'name' => 'Chapter Content',
|
||||
'title' => 'Chapter(s) to include in each work\'s feed entry',
|
||||
'defaultValue' => null,
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'None' => null,
|
||||
'First' => 'first',
|
||||
'Latest' => 'last',
|
||||
'Entire work' => 'all',
|
||||
],
|
||||
],
|
||||
'unique' => [
|
||||
'name' => 'Make separate entries for new fic chapters',
|
||||
'type' => 'checkbox',
|
||||
'required' => false,
|
||||
'title' => 'Make separate entries for new fic chapters',
|
||||
'defaultValue' => 'checked',
|
||||
],
|
||||
'limit' => self::LIMIT,
|
||||
],
|
||||
'Bookmarks' => [
|
||||
'user' => [
|
||||
@ -39,18 +59,13 @@ class AO3Bridge extends BridgeAbstract
|
||||
{
|
||||
switch ($this->queriedContext) {
|
||||
case 'Bookmarks':
|
||||
$user = $this->getInput('user');
|
||||
$this->title = $user;
|
||||
$url = self::URI
|
||||
. '/users/' . $user
|
||||
. '/bookmarks?bookmark_search[sort_column]=bookmarkable_date';
|
||||
$this->collectList($url);
|
||||
$this->collectList($this->getURI());
|
||||
break;
|
||||
case 'List':
|
||||
$this->collectList($this->getInput('url'));
|
||||
$this->collectList($this->getURI());
|
||||
break;
|
||||
case 'Work':
|
||||
$this->collectWork($this->getInput('id'));
|
||||
$this->collectWork($this->getURI());
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -61,9 +76,24 @@ class AO3Bridge extends BridgeAbstract
|
||||
*/
|
||||
private function collectList($url)
|
||||
{
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
$version = 'v0.0.1';
|
||||
$headers = [
|
||||
"useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
|
||||
];
|
||||
$response = getContents($url, $headers);
|
||||
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
|
||||
// Get list title. Will include page range + count in some cases
|
||||
$heading = ($html->find('#main h2', 0));
|
||||
if ($heading->find('a.tag')) {
|
||||
$heading = $heading->find('a.tag', 0);
|
||||
}
|
||||
$this->title = $heading->plaintext;
|
||||
|
||||
$limit = $this->getInput('limit') ?? 3;
|
||||
$count = 0;
|
||||
foreach ($html->find('.index.group > li') as $element) {
|
||||
$item = [];
|
||||
|
||||
@ -72,16 +102,70 @@ class AO3Bridge extends BridgeAbstract
|
||||
continue; // discard deleted works
|
||||
}
|
||||
$item['title'] = $title->plaintext;
|
||||
$item['content'] = $element;
|
||||
$item['uri'] = $title->href;
|
||||
|
||||
$strdate = $element->find('div p.datetime', 0)->plaintext;
|
||||
$item['timestamp'] = strtotime($strdate);
|
||||
|
||||
// detach from rest of page because remove() is buggy
|
||||
$element = str_get_html($element->outertext());
|
||||
$tags = $element->find('ul.required-tags', 0);
|
||||
foreach ($tags->childNodes() as $tag) {
|
||||
$item['categories'][] = html_entity_decode($tag->plaintext);
|
||||
}
|
||||
$tags->remove();
|
||||
$tags = $element->find('ul.tags', 0);
|
||||
foreach ($tags->childNodes() as $tag) {
|
||||
$item['categories'][] = html_entity_decode($tag->plaintext);
|
||||
}
|
||||
$tags->remove();
|
||||
|
||||
$item['content'] = implode('', $element->childNodes());
|
||||
|
||||
$chapters = $element->find('dl dd.chapters', 0);
|
||||
// bookmarked series and external works do not have a chapters count
|
||||
$chapters = (isset($chapters) ? $chapters->plaintext : 0);
|
||||
$item['uid'] = $item['uri'] . "/$strdate/$chapters";
|
||||
if ($this->getInput('unique')) {
|
||||
$item['uid'] = $item['uri'] . "/$strdate/$chapters";
|
||||
} else {
|
||||
$item['uid'] = $item['uri'];
|
||||
}
|
||||
|
||||
|
||||
// Fetch workskin of desired chapter(s) in list
|
||||
if ($this->getInput('range') && ($limit == 0 || $count++ < $limit)) {
|
||||
$url = $item['uri'];
|
||||
switch ($this->getInput('range')) {
|
||||
case ('all'):
|
||||
$url .= '?view_full_work=true';
|
||||
break;
|
||||
case ('first'):
|
||||
break;
|
||||
case ('last'):
|
||||
// only way to get this is using the navigate page unfortunately
|
||||
$url .= '/navigate';
|
||||
$response = getContents($url, $headers);
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
$url = $html->find('ol.index.group > li > a', -1)->href;
|
||||
break;
|
||||
}
|
||||
$response = getContents($url, $headers);
|
||||
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
// remove duplicate fic summary
|
||||
if ($ficsum = $html->find('#workskin > .preface > .summary', 0)) {
|
||||
$ficsum->remove();
|
||||
}
|
||||
$item['content'] .= $html->find('#workskin', 0);
|
||||
}
|
||||
|
||||
// Use predictability of download links to generate enclosures
|
||||
$wid = explode('/', $item['uri'])[4];
|
||||
foreach (['azw3', 'epub', 'mobi', 'pdf', 'html'] as $ext) {
|
||||
$item['enclosures'][] = 'https://archiveofourown.org/downloads/' . $wid . '/work.' . $ext;
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
@ -90,26 +174,31 @@ class AO3Bridge extends BridgeAbstract
|
||||
/**
|
||||
* Feed for recent chapters of a specific work.
|
||||
*/
|
||||
private function collectWork($id)
|
||||
private function collectWork($url)
|
||||
{
|
||||
$url = self::URI . "/works/$id/navigate";
|
||||
$httpClient = RssBridge::getHttpClient();
|
||||
|
||||
$version = 'v0.0.1';
|
||||
$response = $httpClient->request($url, [
|
||||
'useragent' => "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)",
|
||||
]);
|
||||
$headers = [
|
||||
"useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
|
||||
];
|
||||
$response = getContents($url . '/navigate', $headers);
|
||||
|
||||
$html = \str_get_html($response->getBody());
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
|
||||
$response = getContents($url . '?view_full_work=true', $headers);
|
||||
|
||||
$workhtml = \str_get_html($response);
|
||||
$workhtml = defaultLinkTo($workhtml, self::URI);
|
||||
|
||||
$this->title = $html->find('h2 a', 0)->plaintext;
|
||||
|
||||
foreach ($html->find('ol.index.group > li') as $element) {
|
||||
$nav = $html->find('ol.index.group > li');
|
||||
for ($i = 0; $i < count($nav); $i++) {
|
||||
$item = [];
|
||||
|
||||
$element = $nav[$i];
|
||||
$item['title'] = $element->find('a', 0)->plaintext;
|
||||
$item['content'] = $element;
|
||||
$item['content'] = $workhtml->find('#chapter-' . ($i + 1), 0);
|
||||
$item['uri'] = $element->find('a', 0)->href;
|
||||
|
||||
$strdate = $element->find('span.datetime', 0)->plaintext;
|
||||
@ -138,4 +227,24 @@ class AO3Bridge extends BridgeAbstract
|
||||
{
|
||||
return self::URI . '/favicon.ico';
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
$url = parent::getURI();
|
||||
switch ($this->queriedContext) {
|
||||
case 'Bookmarks':
|
||||
$user = $this->getInput('user');
|
||||
$url = self::URI
|
||||
. '/users/' . $user
|
||||
. '/bookmarks?bookmark_search[sort_column]=bookmarkable_date';
|
||||
break;
|
||||
case 'List':
|
||||
$url = $this->getInput('url');
|
||||
break;
|
||||
case 'Work':
|
||||
$url = self::URI . '/works/' . $this->getInput('id');
|
||||
break;
|
||||
}
|
||||
return $url;
|
||||
}
|
||||
}
|
||||
|
@ -40,6 +40,11 @@ class ARDMediathekBridge extends BridgeAbstract
|
||||
* @const IMAGEWIDTHPLACEHOLDER
|
||||
*/
|
||||
const IMAGEWIDTHPLACEHOLDER = '{width}';
|
||||
/**
|
||||
* Title of the current show
|
||||
* @var string
|
||||
*/
|
||||
private $title;
|
||||
|
||||
const PARAMETERS = [
|
||||
[
|
||||
@ -72,7 +77,7 @@ class ARDMediathekBridge extends BridgeAbstract
|
||||
}
|
||||
}
|
||||
|
||||
$url = self::APIENDPOINT . $showID . '/?pageSize=' . self::PAGESIZE;
|
||||
$url = self::APIENDPOINT . $showID . '?pageSize=' . self::PAGESIZE;
|
||||
$rawJSON = getContents($url);
|
||||
$processedJSON = json_decode($rawJSON);
|
||||
|
||||
@ -93,6 +98,17 @@ class ARDMediathekBridge extends BridgeAbstract
|
||||
$this->items[] = $item;
|
||||
}
|
||||
|
||||
$this->title = $processedJSON->title;
|
||||
|
||||
date_default_timezone_set($oldTz);
|
||||
}
|
||||
|
||||
/** {@inheritdoc} */
|
||||
public function getName()
|
||||
{
|
||||
if (!empty($this->title)) {
|
||||
return $this->title;
|
||||
}
|
||||
return parent::getName();
|
||||
}
|
||||
}
|
||||
|
45
bridges/ActivisionResearchBridge.php
Normal file
45
bridges/ActivisionResearchBridge.php
Normal file
@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
class ActivisionResearchBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Activision Research Blog';
|
||||
const URI = 'https://research.activision.com';
|
||||
const DESCRIPTION = 'Posts from the Activision Research blog';
|
||||
const MAINTAINER = 'thefranke';
|
||||
const CACHE_TIMEOUT = 86400; // 24h
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$dom = getSimpleHTMLDOM(static::URI);
|
||||
$dom = $dom->find('div[id="home-blog-feed"]', 0);
|
||||
if (!$dom) {
|
||||
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
|
||||
}
|
||||
$dom = defaultLinkTo($dom, $this->getURI());
|
||||
foreach ($dom->find('div[class="blog-entry"]') as $article) {
|
||||
$a = $article->find('a', 0);
|
||||
|
||||
$blogimg = extractFromDelimiters($article->find('div[class="blog-img"]', 0)->style, 'url(', ')');
|
||||
|
||||
$title = htmlspecialchars_decode($article->find('div[class="title"]', 0)->plaintext);
|
||||
$author = htmlspecialchars_decode($article->find('div[class="author]', 0)->plaintext);
|
||||
$date = $article->find('div[class="pubdate"]', 0)->plaintext;
|
||||
|
||||
$entry = getSimpleHTMLDOMCached($a->href, static::CACHE_TIMEOUT * 7 * 4);
|
||||
$entry = defaultLinkTo($entry, $this->getURI());
|
||||
|
||||
$content = $entry->find('div[class="blog-body"]', 0);
|
||||
$tagsremove = ['script', 'iframe', 'input', 'form'];
|
||||
$content = sanitize($content, $tagsremove);
|
||||
$content = '<img src="' . static::URI . $blogimg . '" alt="">' . $content;
|
||||
|
||||
$this->items[] = [
|
||||
'title' => $title,
|
||||
'author' => $author,
|
||||
'uri' => $a->href,
|
||||
'content' => $content,
|
||||
'timestamp' => strtotime($date),
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
@ -32,8 +32,7 @@ class AirBreizhBridge extends BridgeAbstract
|
||||
public function collectData()
|
||||
{
|
||||
$html = '';
|
||||
$html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'))
|
||||
or returnClientError('No results for this query.');
|
||||
$html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'));
|
||||
|
||||
foreach ($html->find('article') as $article) {
|
||||
$item = [];
|
||||
|
@ -13,12 +13,9 @@ class AllegroBridge extends BridgeAbstract
|
||||
'exampleValue' => 'https://allegro.pl/kategoria/swieze-warzywa-cebula-318660',
|
||||
'required' => true,
|
||||
],
|
||||
'sessioncookie' => [
|
||||
'name' => 'The \'wdctx\' session cookie',
|
||||
'title' => 'Paste the value of the \'wdctx\' cookie from your browser if you want to prevent Allegro imposing rate limits',
|
||||
'pattern' => '^.{70,};?$',
|
||||
// phpcs:ignore
|
||||
'exampleValue' => 'v4.1-oCrmXTMqv2ppC21GTUCKLmUwRPP1ssQVALKuqwsZ1VXjcKgL2vO5TTRM5xMxS9GiyqxF1gAeyc-63dl0coUoBKXCXi_nAmr95yyqGpq2RAFoneZ4L399E8n6iYyemcuGARjAoSfjvLHJCEwvvHHynSgaxlFBu7hUnKfuy39zo9sSQdyTUjotJg3CAZ53q9v2raAnPCyGOAR4ytRILd9p24EJnxp7_oR0XbVPIo1hDa4WmjXFOxph8rHaO5tWd',
|
||||
'cookie' => [
|
||||
'name' => 'The complete cookie value',
|
||||
'title' => 'Paste the value of the cookie value from your browser if you want to prevent Allegro imposing rate limits',
|
||||
'required' => false,
|
||||
],
|
||||
'includeSponsoredOffers' => [
|
||||
@ -70,9 +67,9 @@ class AllegroBridge extends BridgeAbstract
|
||||
|
||||
$opts = [];
|
||||
|
||||
// If a session cookie is provided
|
||||
if ($sessioncookie = $this->getInput('sessioncookie')) {
|
||||
$opts[CURLOPT_COOKIE] = 'wdctx=' . $sessioncookie;
|
||||
// If a cookie is provided
|
||||
if ($cookie = $this->getInput('cookie')) {
|
||||
$opts[CURLOPT_COOKIE] = $cookie;
|
||||
}
|
||||
|
||||
$html = getSimpleHTMLDOM($url, [], $opts);
|
||||
@ -84,11 +81,11 @@ class AllegroBridge extends BridgeAbstract
|
||||
|
||||
$results = $html->find('article[data-analytics-view-custom-context="REGULAR"]');
|
||||
|
||||
if (!$this->getInput('includeSponsoredOffers')) {
|
||||
if ($this->getInput('includeSponsoredOffers')) {
|
||||
$results = array_merge($results, $html->find('article[data-analytics-view-custom-context="SPONSORED"]'));
|
||||
}
|
||||
|
||||
if (!$this->getInput('includePromotedOffers')) {
|
||||
if ($this->getInput('includePromotedOffers')) {
|
||||
$results = array_merge($results, $html->find('article[data-analytics-view-custom-context="PROMOTED"]'));
|
||||
}
|
||||
|
||||
|
@ -146,7 +146,7 @@ EOT;
|
||||
{
|
||||
$uri = $this->getURI();
|
||||
|
||||
return getSimpleHTMLDOM($uri) ?: returnServerError('Could not request Amazon.');
|
||||
return getSimpleHTMLDOM($uri);
|
||||
}
|
||||
|
||||
private function scrapePriceFromMetrics($html)
|
||||
|
278
bridges/AnfrBridge.php
Normal file
278
bridges/AnfrBridge.php
Normal file
@ -0,0 +1,278 @@
|
||||
<?php
|
||||
|
||||
class AnfrBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'ANFR';
|
||||
const URI = 'https://data.anfr.fr/';
|
||||
const DESCRIPTION = 'Fetches data from the French administration "Agence Nationale des Fréquences".';
|
||||
const CACHE_TIMEOUT = 604800; // 7d
|
||||
const MAINTAINER = 'quent1';
|
||||
const PARAMETERS = [
|
||||
'Données sur les réseaux mobiles' => [
|
||||
'departement' => [
|
||||
'name' => 'Département',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'Ain' => '001',
|
||||
'Aisne' => '002',
|
||||
'Allier' => '003',
|
||||
'Alpes-de-Haute-Provence' => '004',
|
||||
'Hautes-Alpes' => '005',
|
||||
'Alpes-Maritimes' => '006',
|
||||
'Ardèche' => '007',
|
||||
'Ardennes' => '008',
|
||||
'Ariège' => '009',
|
||||
'Aube' => '010',
|
||||
'Aude' => '011',
|
||||
'Aveyron' => '012',
|
||||
'Bouches-du-Rhône' => '013',
|
||||
'Calvados' => '014',
|
||||
'Cantal' => '015',
|
||||
'Charente' => '016',
|
||||
'Charente-Maritime' => '017',
|
||||
'Cher' => '018',
|
||||
'Corrèze' => '019',
|
||||
'Corse-du-Sud' => '02A',
|
||||
'Haute-Corse' => '02B',
|
||||
'Côte-d\'Or' => '021',
|
||||
'Côtes-d\'Armor' => '022',
|
||||
'Creuse' => '023',
|
||||
'Dordogne' => '024',
|
||||
'Doubs' => '025',
|
||||
'Drôme' => '026',
|
||||
'Eure' => '027',
|
||||
'Eure-et-Loir' => '028',
|
||||
'Finistère' => '029',
|
||||
'Gard' => '030',
|
||||
'Haute-Garonne' => '031',
|
||||
'Gers' => '032',
|
||||
'Gironde' => '033',
|
||||
'Hérault' => '034',
|
||||
'Ille-et-Vilaine' => '035',
|
||||
'Indre' => '036',
|
||||
'Indre-et-Loire' => '037',
|
||||
'Isère' => '038',
|
||||
'Jura' => '039',
|
||||
'Landes' => '040',
|
||||
'Loir-et-Cher' => '041',
|
||||
'Loire' => '042',
|
||||
'Haute-Loire' => '043',
|
||||
'Loire-Atlantique' => '044',
|
||||
'Loiret' => '045',
|
||||
'Lot' => '046',
|
||||
'Lot-et-Garonne' => '047',
|
||||
'Lozère' => '048',
|
||||
'Maine-et-Loire' => '049',
|
||||
'Manche' => '050',
|
||||
'Marne' => '051',
|
||||
'Haute-Marne' => '052',
|
||||
'Mayenne' => '053',
|
||||
'Meurthe-et-Moselle' => '054',
|
||||
'Meuse' => '055',
|
||||
'Morbihan' => '056',
|
||||
'Moselle' => '057',
|
||||
'Nièvre' => '058',
|
||||
'Nord' => '059',
|
||||
'Oise' => '060',
|
||||
'Orne' => '061',
|
||||
'Pas-de-Calais' => '062',
|
||||
'Puy-de-Dôme' => '063',
|
||||
'Pyrénées-Atlantiques' => '064',
|
||||
'Hautes-Pyrénées' => '065',
|
||||
'Pyrénées-Orientales' => '066',
|
||||
'Bas-Rhin' => '067',
|
||||
'Haut-Rhin' => '068',
|
||||
'Rhône' => '069',
|
||||
'Haute-Saône' => '070',
|
||||
'Saône-et-Loire' => '071',
|
||||
'Sarthe' => '072',
|
||||
'Savoie' => '073',
|
||||
'Haute-Savoie' => '074',
|
||||
'Paris' => '075',
|
||||
'Seine-Maritime' => '076',
|
||||
'Seine-et-Marne' => '077',
|
||||
'Yvelines' => '078',
|
||||
'Deux-Sèvres' => '079',
|
||||
'Somme' => '080',
|
||||
'Tarn' => '081',
|
||||
'Tarn-et-Garonne' => '082',
|
||||
'Var' => '083',
|
||||
'Vaucluse' => '084',
|
||||
'Vendée' => '085',
|
||||
'Vienne' => '086',
|
||||
'Haute-Vienne' => '087',
|
||||
'Vosges' => '088',
|
||||
'Yonne' => '089',
|
||||
'Territoire de Belfort' => '090',
|
||||
'Essonne' => '091',
|
||||
'Hauts-de-Seine' => '092',
|
||||
'Seine-Saint-Denis' => '093',
|
||||
'Val-de-Marne' => '094',
|
||||
'Val-d\'Oise' => '095',
|
||||
'Guadeloupe' => '971',
|
||||
'Martinique' => '972',
|
||||
'Guyane' => '973',
|
||||
'La Réunion' => '974',
|
||||
'Saint-Pierre-et-Miquelon' => '975',
|
||||
'Mayotte' => '976',
|
||||
'Saint-Barthélemy' => '977',
|
||||
'Saint-Martin' => '978',
|
||||
'Terres australes et antarctiques françaises' => '984',
|
||||
'Wallis-et-Futuna' => '986',
|
||||
'Polynésie française' => '987',
|
||||
'Nouvelle-Calédonie' => '988',
|
||||
'Île de Clipperton' => '989'
|
||||
]
|
||||
],
|
||||
'generation' => [
|
||||
'name' => 'Génération',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'2G' => '2G',
|
||||
'3G' => '3G',
|
||||
'4G' => '4G',
|
||||
'5G' => '5G',
|
||||
]
|
||||
],
|
||||
'operateur' => [
|
||||
'name' => 'Opérateur',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'Bouygues Télécom' => 'BOUYGUES TELECOM',
|
||||
'Dauphin Télécom' => 'DAUPHIN TELECOM',
|
||||
'Digiciel' => 'DIGICEL',
|
||||
'Free Caraïbes' => 'FREE CARAIBES',
|
||||
'Free Mobile' => 'FREE MOBILE',
|
||||
'GLOBALTEL' => 'GLOBALTEL',
|
||||
'Office des postes et télécommunications de Nouvelle Calédonie' => 'Gouv Nelle Calédonie (OPT)',
|
||||
'Maore Mobile' => 'MAORE MOBILE',
|
||||
'ONATi' => 'ONATI',
|
||||
'Orange' => 'ORANGE',
|
||||
'Outremer Telecom' => 'OUTREMER TELECOM',
|
||||
'Vodafone polynésie' => 'PMT/VODAPHONE',
|
||||
'SFR' => 'SFR',
|
||||
'SPM Télécom' => 'SPM TELECOM',
|
||||
'Service des Postes et Télécommunications de Polynésie Française' => 'Gouv Nelle Calédonie (OPT)',
|
||||
'SRR' => 'SRR',
|
||||
'Station étrangère' => 'Station étrangère',
|
||||
'Telco OI' => 'TELCO IO',
|
||||
'United Telecommunication Services Caraïbes' => 'UTS Caraibes',
|
||||
'Ora Mobile' => 'VITI SAS',
|
||||
'Zeop' => 'ZEOP'
|
||||
]
|
||||
],
|
||||
'statut' => [
|
||||
'name' => 'Statut',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'En service' => 'En service',
|
||||
'Projet approuvé' => 'Projet approuvé',
|
||||
'Techniquement opérationnel' => 'Techniquement opérationnel',
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$urlParts = [
|
||||
'id' => 'observatoire_2g_3g_4g',
|
||||
'resource_id' => '88ef0887-6b0f-4d3f-8545-6d64c8f597da',
|
||||
'fields' => 'id,adm_lb_nom,sta_nm_dpt,emr_lb_systeme,generation,date_maj,sta_nm_anfr,adr_lb_lieu,adr_lb_add1,adr_lb_add2,adr_lb_add3,adr_nm_cp,statut',
|
||||
'rows' => 10000
|
||||
];
|
||||
|
||||
if (!empty($this->getInput('departement'))) {
|
||||
$urlParts['refine.sta_nm_dpt'] = urlencode($this->getInput('departement'));
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('generation'))) {
|
||||
$urlParts['refine.generation'] = $this->getInput('generation');
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('operateur'))) {
|
||||
// http_build_query() already does urlencoding so this call is redundant
|
||||
$urlParts['refine.adm_lb_nom'] = urlencode($this->getInput('operateur'));
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('statut'))) {
|
||||
$urlParts['refine.statut'] = urlencode($this->getInput('statut'));
|
||||
}
|
||||
|
||||
// API seems to not play well with urlencoded data
|
||||
$url = urljoin(static::URI, '/d4c/api/records/1.0/download/?' . urldecode(http_build_query($urlParts)));
|
||||
|
||||
$json = getContents($url);
|
||||
$data = Json::decode($json, false);
|
||||
$records = $data->records;
|
||||
$frequenciesByStation = [];
|
||||
foreach ($records as $record) {
|
||||
if (!isset($frequenciesByStation[$record->fields->sta_nm_anfr])) {
|
||||
$street = sprintf(
|
||||
'%s %s %s',
|
||||
$record->fields->adr_lb_add1 ?? '',
|
||||
$record->fields->adr_lb_add2 ?? '',
|
||||
$record->fields->adr_lb_add3 ?? ''
|
||||
);
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr] = [
|
||||
'id' => $record->fields->sta_nm_anfr,
|
||||
'operator' => $record->fields->adm_lb_nom,
|
||||
'frequencies' => [],
|
||||
'lastUpdate' => 0,
|
||||
'address' => [
|
||||
'street' => trim($street),
|
||||
'postCode' => $record->fields->adr_nm_cp,
|
||||
'city' => $record->fields->adr_lb_lieu
|
||||
]
|
||||
];
|
||||
}
|
||||
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr]['frequencies'][] = [
|
||||
'generation' => $record->fields->generation,
|
||||
'frequency' => $record->fields->emr_lb_systeme,
|
||||
'status' => $record->fields->statut,
|
||||
'updatedAt' => strtotime($record->fields->date_maj),
|
||||
];
|
||||
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'] = max(
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'],
|
||||
strtotime($record->fields->date_maj)
|
||||
);
|
||||
}
|
||||
|
||||
usort($frequenciesByStation, static fn ($a, $b) => $b['lastUpdate'] <=> $a['lastUpdate']);
|
||||
|
||||
foreach ($frequenciesByStation as $station) {
|
||||
$title = sprintf(
|
||||
'[%s] Mise à jour de la station n°%s à %s (%s)',
|
||||
$station['operator'],
|
||||
$station['id'],
|
||||
$station['address']['city'],
|
||||
$station['address']['postCode']
|
||||
);
|
||||
|
||||
$array_reduce = array_reduce($station['frequencies'], static function ($carry, $frequency) {
|
||||
return sprintf('%s<li>%s : %s</li>', $carry, $frequency['frequency'], $frequency['status']);
|
||||
}, '');
|
||||
|
||||
$content = sprintf(
|
||||
'<h1>Adresse complète</h1><p>%s<br>%s<br>%s</p><h1>Fréquences</h1><p><ul>%s</ul></p>',
|
||||
$station['address']['street'],
|
||||
$station['address']['postCode'],
|
||||
$station['address']['city'],
|
||||
$array_reduce
|
||||
);
|
||||
|
||||
$this->items[] = [
|
||||
'uid' => $station['id'],
|
||||
'timestamp' => $station['lastUpdate'],
|
||||
'title' => $title,
|
||||
'content' => $content,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
87
bridges/AnisearchBridge.php
Normal file
87
bridges/AnisearchBridge.php
Normal file
@ -0,0 +1,87 @@
|
||||
<?php
|
||||
|
||||
class AnisearchBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'Tone866';
|
||||
const NAME = 'Anisearch';
|
||||
const URI = 'https://www.anisearch.de';
|
||||
const CACHE_TIMEOUT = 1800; // 30min
|
||||
const DESCRIPTION = 'Feed for Anisearch';
|
||||
const PARAMETERS = [[
|
||||
'category' => [
|
||||
'name' => 'Dub',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'DE'
|
||||
=> 'https://www.anisearch.de/anime/index/page-1?char=all&synchro=de&sort=date&order=desc&view=4',
|
||||
'EN'
|
||||
=> 'https://www.anisearch.de/anime/index/page-1?char=all&synchro=en&sort=date&order=desc&view=4',
|
||||
'JP'
|
||||
=> 'https://www.anisearch.de/anime/index/page-1?char=all&synchro=ja&sort=date&order=desc&view=4'
|
||||
]
|
||||
],
|
||||
'trailers' => [
|
||||
'name' => 'Trailers',
|
||||
'type' => 'checkbox',
|
||||
'title' => 'Will include trailes',
|
||||
'defaultValue' => false
|
||||
]
|
||||
]];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$baseurl = 'https://www.anisearch.de/';
|
||||
$trailers = false;
|
||||
$trailers = $this->getInput('trailers');
|
||||
$limit = 10;
|
||||
if ($trailers) {
|
||||
$limit = 5;
|
||||
}
|
||||
|
||||
$dom = getSimpleHTMLDOM($this->getInput('category'));
|
||||
|
||||
foreach ($dom->find('li.btype0') as $key => $li) {
|
||||
if ($key >= $limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
$a = $li->find('a', 0);
|
||||
$title = $a->find('span.title', 0);
|
||||
$url = $baseurl . $a->href;
|
||||
|
||||
//get article
|
||||
$domarticle = getSimpleHTMLDOM($url);
|
||||
$content = $domarticle->find('div.details-text', 0);
|
||||
|
||||
//get header-image and set absolute src
|
||||
$headerimage = $domarticle->find('img#details-cover', 0);
|
||||
$src = $headerimage->src;
|
||||
|
||||
foreach ($content->find('.hidden') as $element) {
|
||||
$element->remove();
|
||||
}
|
||||
|
||||
//get trailer
|
||||
$ytlink = '';
|
||||
if ($trailers) {
|
||||
$trailerlink = $domarticle->find('section#trailers > div > div.swiper > ul.swiper-wrapper > li.swiper-slide > a', 0);
|
||||
if (isset($trailerlink)) {
|
||||
$trailersite = getSimpleHTMLDOM($baseurl . $trailerlink->href);
|
||||
$trailer = $trailersite->find('div#video > iframe', 0);
|
||||
$trailer = $trailer->{'data-xsrc'};
|
||||
$ytlink = <<<EOT
|
||||
<br /><iframe width="560" height="315" src="$trailer" title="YouTube video player"
|
||||
frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
|
||||
referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
|
||||
EOT;
|
||||
}
|
||||
}
|
||||
|
||||
$this->items[] = [
|
||||
'title' => $title->plaintext,
|
||||
'uri' => $url,
|
||||
'content' => $headerimage . '<br />' . $content . $ytlink
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
@ -126,28 +126,36 @@ class AnnasArchiveBridge extends BridgeAbstract
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($list->find('.w-full > .mb-4 > div > a') as $element) {
|
||||
$item = [];
|
||||
$item['title'] = $element->find('h3', 0)->plaintext;
|
||||
$item['author'] = $element->find('div.italic', 0)->plaintext;
|
||||
$item['uri'] = $element->href;
|
||||
$item['content'] = $element->plaintext;
|
||||
$item['uid'] = $item['uri'];
|
||||
$elements = $list->find('.w-full > .mb-4 > div');
|
||||
foreach ($elements as $element) {
|
||||
// stop added entries once partial match list starts
|
||||
if (str_contains($element->innertext, 'partial match')) {
|
||||
break;
|
||||
}
|
||||
if ($element = $element->find('a', 0)) {
|
||||
$item = [];
|
||||
$item['title'] = $element->find('h3', 0)->plaintext;
|
||||
$item['author'] = $element->find('div.italic', 0)->plaintext;
|
||||
$item['uri'] = $element->href;
|
||||
$item['content'] = $element->plaintext;
|
||||
$item['uid'] = $item['uri'];
|
||||
|
||||
if ($item_html = getSimpleHTMLDOMCached($item['uri'])) {
|
||||
$item_html = defaultLinkTo($item_html, self::URI);
|
||||
$item['content'] .= $item_html->find('main img', 0);
|
||||
$item['content'] .= $item_html->find('main .mt-4', 0); // Summary
|
||||
if ($links = $item_html->find('main ul.mb-4', -1)) {
|
||||
foreach ($links->find('li > a.js-download-link') as $file) {
|
||||
$item['enclosures'][] = $file->href;
|
||||
$item_html = getSimpleHTMLDOMCached($item['uri'], 86400 * 20);
|
||||
if ($item_html) {
|
||||
$item_html = defaultLinkTo($item_html, self::URI);
|
||||
$item['content'] .= $item_html->find('main img', 0);
|
||||
$item['content'] .= $item_html->find('main .mt-4', 0); // Summary
|
||||
foreach ($item_html->find('main ul.mb-4 > li > a.js-download-link') as $file) {
|
||||
if (!str_contains($file->href, 'fast_download')) {
|
||||
$item['enclosures'][] = $file->href;
|
||||
}
|
||||
}
|
||||
// Remove bulk torrents from enclosures list
|
||||
$item['enclosures'] = array_diff($item['enclosures'], [self::URI . 'datasets']);
|
||||
}
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -18,9 +18,45 @@ class AppleMusicBridge extends BridgeAbstract
|
||||
'required' => true,
|
||||
],
|
||||
]];
|
||||
const CACHE_TIMEOUT = 21600; // 6 hours
|
||||
const CACHE_TIMEOUT = 60 * 60 * 6; // 6 hours
|
||||
|
||||
private $title;
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$items = $this->getJson();
|
||||
$artist = $this->getArtist($items);
|
||||
|
||||
$this->title = $artist->artistName;
|
||||
|
||||
foreach ($items as $item) {
|
||||
if ($item->wrapperType === 'collection') {
|
||||
$copyright = $item->copyright ?? '';
|
||||
$artworkUrl500 = str_replace('/100x100', '/500x500', $item->artworkUrl100);
|
||||
$artworkUrl2000 = str_replace('/100x100', '/2000x2000', $item->artworkUrl100);
|
||||
$escapedCollectionName = htmlspecialchars($item->collectionName);
|
||||
|
||||
$this->items[] = [
|
||||
'title' => $item->collectionName,
|
||||
'uri' => $item->collectionViewUrl,
|
||||
'timestamp' => $item->releaseDate,
|
||||
'enclosures' => $artworkUrl500,
|
||||
'author' => $item->artistName,
|
||||
'content' => "<figure>
|
||||
<img srcset=\"$item->artworkUrl60 60w, $item->artworkUrl100 100w, $artworkUrl500 500w, $artworkUrl2000 2000w\"
|
||||
sizes=\"100%\" src=\"$artworkUrl2000\"
|
||||
alt=\"Cover of $escapedCollectionName\"
|
||||
style=\"display: block; margin: 0 auto;\" />
|
||||
<figcaption>
|
||||
from <a href=\"$artist->artistLinkUrl\">$item->artistName</a><br />$copyright
|
||||
</figcaption>
|
||||
</figure>",
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function getJson()
|
||||
{
|
||||
# Limit the amount of releases to 50
|
||||
if ($this->getInput('limit') > 50) {
|
||||
@ -29,31 +65,53 @@ class AppleMusicBridge extends BridgeAbstract
|
||||
$limit = $this->getInput('limit');
|
||||
}
|
||||
|
||||
$url = 'https://itunes.apple.com/lookup?id='
|
||||
. $this->getInput('artist')
|
||||
. '&entity=album&limit='
|
||||
. $limit .
|
||||
'&sort=recent';
|
||||
$url = 'https://itunes.apple.com/lookup?id=' . $this->getInput('artist') . '&entity=album&limit=' . $limit . '&sort=recent';
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
|
||||
$json = json_decode($html);
|
||||
$result = $json->results;
|
||||
|
||||
foreach ($json->results as $obj) {
|
||||
if ($obj->wrapperType === 'collection') {
|
||||
$copyright = $obj->copyright ?? '';
|
||||
|
||||
$this->items[] = [
|
||||
'title' => $obj->artistName . ' - ' . $obj->collectionName,
|
||||
'uri' => $obj->collectionViewUrl,
|
||||
'timestamp' => $obj->releaseDate,
|
||||
'enclosures' => $obj->artworkUrl100,
|
||||
'content' => '<a href=' . $obj->collectionViewUrl
|
||||
. '><img src="' . $obj->artworkUrl100 . '" /></a><br><br>'
|
||||
. $obj->artistName . ' - ' . $obj->collectionName
|
||||
. '<br>'
|
||||
. $copyright,
|
||||
];
|
||||
}
|
||||
if (!is_array($result) || count($result) == 0) {
|
||||
returnServerError('There is no artist with id "' . $this->getInput('artist') . '".');
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function getArtist($json)
|
||||
{
|
||||
$nameArray = array_filter($json, function ($obj) {
|
||||
return $obj->wrapperType == 'artist';
|
||||
});
|
||||
|
||||
if (count($nameArray) === 1) {
|
||||
return $nameArray[0];
|
||||
}
|
||||
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (isset($this->title)) {
|
||||
return $this->title;
|
||||
}
|
||||
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
if (empty($this->getInput('artist'))) {
|
||||
return parent::getIcon();
|
||||
}
|
||||
|
||||
// it isn't necessary to set the correct artist name into the url
|
||||
$url = 'https://music.apple.com/us/artist/jon-bellion/' . $this->getInput('artist');
|
||||
$html = getSimpleHTMLDOMCached($url);
|
||||
$image = $html->find('meta[property="og:image"]', 0)->content;
|
||||
|
||||
$imageUpdatedSize = preg_replace('/\/\d*x\d*cw/i', '/144x144-999', $image);
|
||||
|
||||
return $imageUpdatedSize;
|
||||
}
|
||||
}
|
||||
|
@ -37,35 +37,82 @@ class ArsTechnicaBridge extends FeedExpander
|
||||
{
|
||||
$item_html = getSimpleHTMLDOMCached($item['uri']);
|
||||
$item_html = defaultLinkTo($item_html, self::URI);
|
||||
$item['content'] = $item_html->find('.article-content', 0);
|
||||
|
||||
$pages = $item_html->find('nav.page-numbers > .numbers > a', -2);
|
||||
if (null !== $pages) {
|
||||
for ($i = 2; $i <= $pages->innertext; $i++) {
|
||||
$page_url = $item['uri'] . '&page=' . $i;
|
||||
$page_html = getSimpleHTMLDOMCached($page_url);
|
||||
$page_html = defaultLinkTo($page_html, self::URI);
|
||||
$item['content'] .= $page_html->find('.article-content', 0);
|
||||
$content = '';
|
||||
$header = $item_html->find('article header', 0);
|
||||
$leading = $header->find('p[class*=leading]', 0);
|
||||
if ($leading != null) {
|
||||
$content .= '<p>' . $leading->innertext . '</p>';
|
||||
}
|
||||
$intro_image = $header->find('img.intro-image', 0);
|
||||
if ($intro_image != null) {
|
||||
$content .= '<figure>' . $intro_image;
|
||||
|
||||
$image_caption = $header->find('.caption .caption-content', 0);
|
||||
if ($image_caption != null) {
|
||||
$content .= '<figcaption>' . $image_caption->innertext . '</figcaption>';
|
||||
}
|
||||
$item['content'] = str_get_html($item['content']);
|
||||
$content .= '</figure>';
|
||||
}
|
||||
|
||||
foreach ($item_html->find('.post-content') as $content_tag) {
|
||||
$content .= $content_tag->innertext;
|
||||
}
|
||||
|
||||
$item['content'] = str_get_html($content);
|
||||
|
||||
$parsely = $item_html->find('[name="parsely-page"]', 0);
|
||||
$parsely_json = json_decode(html_entity_decode($parsely->content), true);
|
||||
$item['categories'] = $parsely_json['tags'];
|
||||
|
||||
// Some lightboxes are nested in figures. I'd guess that's a
|
||||
// bug in the website
|
||||
foreach ($item['content']->find('figure div div.ars-lightbox') as $weird_lightbox) {
|
||||
$weird_lightbox->parent->parent->outertext = $weird_lightbox;
|
||||
}
|
||||
|
||||
// It's easier to reconstruct the whole thing than remove
|
||||
// duplicate reactive tags
|
||||
foreach ($item['content']->find('.ars-lightbox') as $lightbox) {
|
||||
$lightbox_content = '';
|
||||
foreach ($lightbox->find('.ars-lightbox-item') as $lightbox_item) {
|
||||
$img = $lightbox_item->find('img', 0);
|
||||
if ($img != null) {
|
||||
$lightbox_content .= '<figure>' . $img;
|
||||
$caption = $lightbox_item->find('div.pswp-caption-content', 0);
|
||||
if ($caption != null) {
|
||||
$credit = $lightbox_item->find('div.ars-gallery-caption-credit', 0);
|
||||
if ($credit != null) {
|
||||
$credit->innertext = 'Credit: ' . $credit->innertext;
|
||||
}
|
||||
$lightbox_content .= '<figcaption>' . $caption->innertext . '</figcaption>';
|
||||
}
|
||||
$lightbox_content .= '</figure>';
|
||||
}
|
||||
}
|
||||
$lightbox->innertext = $lightbox_content;
|
||||
}
|
||||
|
||||
// remove various ars advertising
|
||||
$item['content']->find('#social-left', 0)->remove();
|
||||
foreach ($item['content']->find('.ars-component-buy-box') as $ad) {
|
||||
foreach ($item['content']->find('.ars-interlude-container') as $ad) {
|
||||
$ad->remove();
|
||||
}
|
||||
foreach ($item['content']->find('.ad_wrapper') as $ad) {
|
||||
$ad->remove();
|
||||
foreach ($item['content']->find('.toc-container') as $toc) {
|
||||
$toc->remove();
|
||||
}
|
||||
foreach ($item['content']->find('.sidebar') as $ad) {
|
||||
$ad->remove();
|
||||
|
||||
// Mostly YouTube videos
|
||||
$iframes = $item['content']->find('iframe');
|
||||
foreach ($iframes as $iframe) {
|
||||
$iframe->outertext = '<a href="' . $iframe->src . '">' . $iframe->src . '</a>';
|
||||
}
|
||||
// This fixed padding around the former iframes and actual inline videos
|
||||
foreach ($item['content']->find('div[style*=aspect-ratio]') as $styled) {
|
||||
$styled->removeAttribute('style');
|
||||
}
|
||||
|
||||
$item['content'] = backgroundToImg($item['content']);
|
||||
|
||||
$item['uid'] = explode('=', $item['uri'])[1];
|
||||
|
||||
$item['uid'] = strval($parsely_json['post_id']);
|
||||
return $item;
|
||||
}
|
||||
}
|
||||
|
@ -45,7 +45,6 @@ class AsahiShimbunAJWBridge extends BridgeAbstract
|
||||
|
||||
foreach ($html->find('#MainInner li a') as $element) {
|
||||
if ($element->parent()->class == 'HeadlineTopImage-S') {
|
||||
Debug::log('Skip Headline, it is repeated below');
|
||||
continue;
|
||||
}
|
||||
$item = [];
|
||||
|
@ -105,8 +105,7 @@ class AssociatedPressNewsBridge extends BridgeAbstract
|
||||
|
||||
private function collectCardData()
|
||||
{
|
||||
$json = getContents($this->getTagURI())
|
||||
or returnServerError('Could not request: ' . $this->getTagURI());
|
||||
$json = getContents($this->getTagURI());
|
||||
|
||||
$tagContents = json_decode($json, true);
|
||||
|
||||
|
344
bridges/AuctionetBridge.php
Normal file
344
bridges/AuctionetBridge.php
Normal file
@ -0,0 +1,344 @@
|
||||
<?php
|
||||
|
||||
class AuctionetBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Auctionet';
|
||||
const URI = 'https://www.auctionet.com';
|
||||
const DESCRIPTION = 'Fetches info about auction objects from Auctionet (an auction platform for many European auction houses)';
|
||||
const MAINTAINER = 'Qluxzz';
|
||||
const PARAMETERS = [[
|
||||
'category' => [
|
||||
'name' => 'Category',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'All categories' => '',
|
||||
'Art' => [
|
||||
'All' => '25-art',
|
||||
'Drawings' => '119-drawings',
|
||||
'Engravings & Prints' => '27-engravings-prints',
|
||||
'Other' => '30-other',
|
||||
'Paintings' => '28-paintings',
|
||||
'Photography' => '26-photography',
|
||||
'Sculptures & Bronzes' => '29-sculptures-bronzes',
|
||||
],
|
||||
'Asiatica' => [
|
||||
'All' => '117-asiatica',
|
||||
],
|
||||
'Books, Maps & Manuscripts' => [
|
||||
'All' => '50-books-maps-manuscripts',
|
||||
'Autographs & Manuscripts' => '206-autographs-manuscripts',
|
||||
'Books' => '204-books',
|
||||
'Maps' => '205-maps',
|
||||
'Other' => '207-other',
|
||||
],
|
||||
'Carpets & Textiles' => [
|
||||
'All' => '35-carpets-textiles',
|
||||
'Carpets' => '36-carpets',
|
||||
'Textiles' => '37-textiles',
|
||||
],
|
||||
'Ceramics & Porcelain' => [
|
||||
'All' => '9-ceramics-porcelain',
|
||||
'European' => '10-european',
|
||||
'Oriental' => '11-oriental',
|
||||
'Rest of the world' => '12-rest-of-the-world',
|
||||
'Tableware' => '210-tableware',
|
||||
],
|
||||
'Clocks & Watches' => [
|
||||
'All' => '31-clocks-watches',
|
||||
'Carriage & Miniature Clocks' => '258-carriage-miniature-clocks',
|
||||
'Longcase clocks' => '32-longcase-clocks',
|
||||
'Mantel clocks' => '33-mantel-clocks',
|
||||
'Other clocks' => '34-other-clocks',
|
||||
'Pocket & Stop Watches' => '110-pocket-stop-watches',
|
||||
'Wall Clocks' => '127-wall-clocks',
|
||||
'Wristwatches' => '15-wristwatches',
|
||||
],
|
||||
'Coins, Medals & Stamps' => [
|
||||
'All' => '46-coins-medals-stamps',
|
||||
'Coins' => '128-coins',
|
||||
'Orders & Medals' => '135-orders-medals',
|
||||
'Other' => '131-other',
|
||||
'Stamps' => '136-stamps',
|
||||
],
|
||||
'Folk art' => [
|
||||
'All' => '58-folk-art',
|
||||
'Bowls & Boxes' => '121-bowls-boxes',
|
||||
'Furniture' => '122-furniture',
|
||||
'Other' => '123-other',
|
||||
'Tools & Gears' => '120-tools-gears',
|
||||
],
|
||||
'Furniture' => [
|
||||
'All' => '16-furniture',
|
||||
'Armchairs & Chairs' => '18-armchairs-chairs',
|
||||
'Chests of drawers' => '24-chests-of-drawers',
|
||||
'Cupboards, Cabinets & Shelves' => '23-cupboards-cabinets-shelves',
|
||||
'Dining room furniture' => '22-dining-room-furniture',
|
||||
'Garden' => '21-garden',
|
||||
'Other' => '17-other',
|
||||
'Sofas & seatings' => '20-sofas-seatings',
|
||||
'Tables' => '19-tables',
|
||||
],
|
||||
'Glass' => [
|
||||
'All' => '6-glass',
|
||||
'Art glass' => '208-art-glass',
|
||||
'Other' => '8-other',
|
||||
'Tableware' => '7-tableware',
|
||||
'Utility glass' => '209-utility-glass',
|
||||
],
|
||||
'Jewellery & Gemstones' => [
|
||||
'All' => '13-jewellery-gemstones',
|
||||
'Alliance rings' => '113-alliance-rings',
|
||||
'Bracelets' => '106-bracelets',
|
||||
'Brooches & Pendants' => '107-brooches-pendants',
|
||||
'Costume Jewellery' => '259-costume-jewellery',
|
||||
'Cufflinks & Tie Pins' => '111-cufflinks-tie-pins',
|
||||
'Ear studs' => '116-ear-studs',
|
||||
'Earrings' => '115-earrings',
|
||||
'Gemstones' => '48-gemstones',
|
||||
'Jewellery' => '14-jewellery',
|
||||
'Jewellery Suites' => '109-jewellery-suites',
|
||||
'Necklace' => '104-necklace',
|
||||
'Other' => '118-other',
|
||||
'Rings' => '112-rings',
|
||||
'Signet rings' => '105-signet-rings',
|
||||
'Solitaire rings' => '114-solitaire-rings',
|
||||
],
|
||||
'Licence weapons' => [
|
||||
'All' => '59-licence-weapons',
|
||||
'Combi/Combo' => '63-combi-combo',
|
||||
'Double express rifles' => '60-double-express-rifles',
|
||||
'Rifles' => '61-rifles',
|
||||
'Shotguns' => '62-shotguns',
|
||||
],
|
||||
'Lighting & Lamps' => [
|
||||
'All' => '1-lighting-lamps',
|
||||
'Candlesticks' => '4-candlesticks',
|
||||
'Ceiling lights' => '3-ceiling-lights',
|
||||
'Chandeliers' => '203-chandeliers',
|
||||
'Floor lights' => '2-floor-lights',
|
||||
'Other lighting' => '5-other-lighting',
|
||||
'Table Lamps' => '125-table-lamps',
|
||||
'Wall Lights' => '124-wall-lights',
|
||||
],
|
||||
'Mirrors' => [
|
||||
'All' => '42-mirrors',
|
||||
],
|
||||
'Miscellaneous' => [
|
||||
'All' => '43-miscellaneous',
|
||||
'Fishing equipment' => '54-fishing-equipment',
|
||||
'Miscellaneous' => '47-miscellaneous',
|
||||
'Modern Tools' => '133-modern-tools',
|
||||
'Modern consumer electronics' => '52-modern-consumer-electronics',
|
||||
'Musical instruments' => '51-musical-instruments',
|
||||
'Technica & Nautica' => '45-technica-nautica',
|
||||
],
|
||||
'Photo, Cameras & Lenses' => [
|
||||
'All' => '57-photo-cameras-lenses',
|
||||
'Cameras & accessories' => '71-cameras-accessories',
|
||||
'Optics' => '66-optics',
|
||||
'Other' => '72-other',
|
||||
],
|
||||
'Silver & Metals' => [
|
||||
'All' => '38-silver-metals',
|
||||
'Other metals' => '40-other-metals',
|
||||
'Pewter, Brass & Copper' => '41-pewter-brass-copper',
|
||||
'Silver' => '39-silver',
|
||||
'Silver plated' => '213-silver-plated',
|
||||
],
|
||||
'Toys' => [
|
||||
'All' => '44-toys',
|
||||
'Comics' => '211-comics',
|
||||
'Toys' => '212-toys',
|
||||
],
|
||||
'Tribal art' => [
|
||||
'All' => '134-tribal-art',
|
||||
],
|
||||
'Vehicles, Boats & Parts' => [
|
||||
'All' => '249-vehicles-boats-parts',
|
||||
'Automobilia & Transport' => '255-automobilia-transport',
|
||||
'Bicycles' => '132-bicycles',
|
||||
'Boats & Accessories' => '250-boats-accessories',
|
||||
'Car parts' => '253-car-parts',
|
||||
'Cars' => '215-cars',
|
||||
'Moped parts' => '254-moped-parts',
|
||||
'Mopeds' => '216-mopeds',
|
||||
'Motorcycle parts' => '252-motorcycle-parts',
|
||||
'Motorcycles' => '251-motorcycles',
|
||||
'Other' => '256-other',
|
||||
],
|
||||
'Vintage & Designer Fashion' => [
|
||||
'All' => '49-vintage-designer-fashion',
|
||||
],
|
||||
'Weapons & Militaria' => [
|
||||
'All' => '137-weapons-militaria',
|
||||
'Airguns' => '257-airguns',
|
||||
'Armour & Uniform' => '138-armour-uniform',
|
||||
'Edged weapons' => '130-edged-weapons',
|
||||
'Guns & Rifles' => '129-guns-rifles',
|
||||
'Other' => '214-other',
|
||||
],
|
||||
'Wine, Port & Spirits' => [
|
||||
'All' => '170-wine-port-spirits',
|
||||
],
|
||||
]
|
||||
],
|
||||
'sort_order' => [
|
||||
'name' => 'Sort order',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Most bids' => 'bids_count_desc',
|
||||
'Lowest bid' => 'bid_asc',
|
||||
'Highest bid' => 'bid_desc',
|
||||
'Last bid on' => 'bid_on',
|
||||
'Ending soonest' => 'end_asc_active',
|
||||
'Lowest estimate' => 'estimate_asc',
|
||||
'Highest estimate' => 'estimate_desc',
|
||||
'Recently added' => 'recent'
|
||||
],
|
||||
],
|
||||
'country' => [
|
||||
'name' => 'Country',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'All' => '',
|
||||
'Denmark' => 'DK',
|
||||
'Finland' => 'FI',
|
||||
'Germany' => 'DE',
|
||||
'Spain' => 'ES',
|
||||
'Sweden' => 'SE',
|
||||
'United Kingdom' => 'GB'
|
||||
]
|
||||
],
|
||||
'language' => [
|
||||
'name' => 'Language',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'English' => 'en',
|
||||
'Español' => 'es',
|
||||
'Deutsch' => 'de',
|
||||
'Svenska' => 'sv',
|
||||
'Dansk' => 'da',
|
||||
'Suomi' => 'fi',
|
||||
],
|
||||
],
|
||||
]];
|
||||
|
||||
const CACHE_TIMEOUT = 3600; // 1 hour
|
||||
|
||||
private $title;
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
// Each page contains 48 auctions
|
||||
// So we fetch 10 pages so we decrease the likelihood
|
||||
// of missing auctions between feed refreshes
|
||||
|
||||
// Fetch first page and use that to get title
|
||||
{
|
||||
$url = $this->getUrl(1);
|
||||
$data = getContents($url);
|
||||
|
||||
$title = $this->getDocumentTitle($data);
|
||||
|
||||
$this->items = array_merge($this->items, $this->parsePageData($data));
|
||||
}
|
||||
|
||||
// Fetch remaining pages
|
||||
for ($page = 2; $page <= 10; $page++) {
|
||||
$url = $this->getUrl($page);
|
||||
|
||||
$data = getContents($url);
|
||||
|
||||
$this->items = array_merge($this->items, $this->parsePageData($data));
|
||||
}
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
return $this->title ?: parent::getName();
|
||||
}
|
||||
|
||||
|
||||
/* HELPERS */
|
||||
|
||||
private function getUrl($page)
|
||||
{
|
||||
$category = $this->getInput('category');
|
||||
$language = $this->getInput('language');
|
||||
$sort_order = $this->getInput('sort_order');
|
||||
$country = $this->getInput('country');
|
||||
|
||||
$url = self::URI . '/' . $language . '/search';
|
||||
|
||||
if ($category) {
|
||||
$url = $url . '/' . $category;
|
||||
}
|
||||
|
||||
$query = [];
|
||||
$query['page'] = $page;
|
||||
|
||||
if ($sort_order) {
|
||||
$query['order'] = $sort_order;
|
||||
}
|
||||
|
||||
if ($country) {
|
||||
$query['country_code'] = $country;
|
||||
}
|
||||
|
||||
if (count($query) > 0) {
|
||||
$url = $url . '?' . http_build_query($query);
|
||||
}
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function getDocumentTitle($data)
|
||||
{
|
||||
$title_elem = '<title>';
|
||||
$title_elem_length = strlen($title_elem);
|
||||
$title_start = strpos($data, $title_elem);
|
||||
$title_end = strpos($data, '</title>', $title_start);
|
||||
$title_length = $title_end - $title_start + strlen($title_elem);
|
||||
$title = substr($data, $title_start + strlen($title_elem), $title_length);
|
||||
|
||||
return $title;
|
||||
}
|
||||
|
||||
/**
|
||||
* The auction items data is included in the HTML document
|
||||
* as a HTML entities encoded JSON structure
|
||||
* which is used to hydrate the React component for the list of auctions
|
||||
*/
|
||||
private function parsePageData($data)
|
||||
{
|
||||
$key = 'data-react-props="';
|
||||
$keyLength = strlen($key);
|
||||
|
||||
$start = strpos($data, $key);
|
||||
$end = strpos($data, '"', $start + strlen($key));
|
||||
$length = $end - ($start + $keyLength);
|
||||
|
||||
$jsonString = substr($data, $start + $keyLength, $length);
|
||||
|
||||
$jsonData = json_decode(htmlspecialchars_decode($jsonString), false);
|
||||
|
||||
$items = [];
|
||||
|
||||
foreach ($jsonData->{'items'} as $item) {
|
||||
$title = $item->{'longTitle'};
|
||||
$relative_url = $item->{'url'};
|
||||
$images = $item->{'imageUrls'};
|
||||
$id = $item->{'auctionId'};
|
||||
|
||||
$items[] = [
|
||||
'title' => $title,
|
||||
'uri' => self::URI . $relative_url,
|
||||
'uid' => $id,
|
||||
'content' => count($images) > 0 ? "<img src='$images[0]'/><br/>$title" : $title,
|
||||
'enclosures' => array_slice($images, 1),
|
||||
];
|
||||
}
|
||||
|
||||
return $items;
|
||||
}
|
||||
}
|
@ -29,7 +29,7 @@ class BAEBridge extends BridgeAbstract
|
||||
public function collectData()
|
||||
{
|
||||
$url = $this->getURI();
|
||||
$html = getSimpleHTMLDOM($url) or returnClientError('No results for this query.');
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
|
||||
$annonces = $html->find('main article');
|
||||
foreach ($annonces as $annonce) {
|
||||
|
@ -8,6 +8,7 @@ class BMDSystemhausBlogBridge extends BridgeAbstract
|
||||
const URI = 'https://www.bmd.com';
|
||||
const DONATION_URI = 'https://paypal.me/cntools';
|
||||
const DESCRIPTION = 'BMD Systemhaus - We make business easy';
|
||||
const BMD_FAV_ICON = 'https://www.bmd.com/favicon.ico';
|
||||
|
||||
const ITEMSTYLE = [
|
||||
'ilcr' => '<table width="100%"><tr><td style="vertical-align: top;">{data_img}</td><td style="vertical-align: top;">{data_content}</td></tr></table>',
|
||||
@ -53,7 +54,7 @@ class BMDSystemhausBlogBridge extends BridgeAbstract
|
||||
public function collectData()
|
||||
{
|
||||
// get website content
|
||||
$html = getSimpleHTMLDOM($this->getURI()) or returnServerError('No contents received!');
|
||||
$html = getSimpleHTMLDOM($this->getURI());
|
||||
|
||||
// Convert relative links in HTML into absolute links
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
@ -148,32 +149,73 @@ class BMDSystemhausBlogBridge extends BridgeAbstract
|
||||
return null;
|
||||
}
|
||||
|
||||
if ($parsedUrl->getHost() != 'www.bmd.com') {
|
||||
if (!in_array($parsedUrl->getHost(), ['www.bmd.com', 'bmd.com'])) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$path = explode('/', $parsedUrl->getPath());
|
||||
$lang = '';
|
||||
|
||||
if ($this->getURIbyCountry($path[1]) == '') {
|
||||
return null;
|
||||
// extract language from url
|
||||
$path = explode('/', $parsedUrl->getPath());
|
||||
if (count($path) > 1) {
|
||||
$lang = $path[1];
|
||||
|
||||
// validate data
|
||||
if ($this->getURIbyCountry($lang) == '') {
|
||||
$lang = '';
|
||||
}
|
||||
}
|
||||
|
||||
// if no country available, find language by browser
|
||||
if ($lang == '') {
|
||||
$srvLanguages = explode(';', $_SERVER['HTTP_ACCEPT_LANGUAGE']);
|
||||
if (count($srvLanguages) > 0) {
|
||||
$languages = explode(',', $srvLanguages[0]);
|
||||
if (count($languages) > 0) {
|
||||
for ($i = 0; $i < count($languages); $i++) {
|
||||
$langDetails = explode('-', $languages[$i]);
|
||||
if (count($langDetails) > 1) {
|
||||
$lang = $langDetails[1];
|
||||
} else {
|
||||
$lang = substr($srvLanguages[0], 0, 2);
|
||||
}
|
||||
|
||||
// validate data
|
||||
if ($this->getURIbyCountry($lang) == '') {
|
||||
$lang = '';
|
||||
}
|
||||
|
||||
if ($lang != '') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if no URL found by language, use AT as default
|
||||
if ($this->getURIbyCountry($lang) == '') {
|
||||
$lang = 'at';
|
||||
}
|
||||
|
||||
$params = [];
|
||||
$params['country'] = $path[1];
|
||||
$params['country'] = strtolower($lang);
|
||||
|
||||
return $params;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------
|
||||
public function getURI()
|
||||
{
|
||||
$lURI = $this->getURIbyCountry($this->getInput('country'));
|
||||
$country = $this->getInput('country') ?? '';
|
||||
$lURI = $this->getURIbyCountry($country);
|
||||
return $lURI != '' ? $lURI : parent::getURI();
|
||||
}
|
||||
|
||||
//-----------------------------------------------------
|
||||
public function getIcon()
|
||||
{
|
||||
return 'https://www.bmd.com/favicon.ico';
|
||||
return self::BMD_FAV_ICON;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------
|
||||
@ -192,7 +234,7 @@ class BMDSystemhausBlogBridge extends BridgeAbstract
|
||||
//-----------------------------------------------------
|
||||
private function getURIbyCountry($country)
|
||||
{
|
||||
switch ($country) {
|
||||
switch (strtolower($country)) {
|
||||
case 'at':
|
||||
return 'https://www.bmd.com/at/ueber-bmd/blog-ohne-filter.html';
|
||||
case 'de':
|
||||
|
@ -284,8 +284,7 @@ class BadDragonBridge extends BridgeAbstract
|
||||
case 'Clearance':
|
||||
$toyData = json_decode(getContents($this->inputToURL(true)));
|
||||
|
||||
$productList = json_decode(getContents(self::URI
|
||||
. 'api/inventory-toy/product-list'));
|
||||
$productList = json_decode(getContents(self::URI . 'api/inventory-toy/product-list'));
|
||||
|
||||
foreach ($toyData->toys as $toy) {
|
||||
$item = [];
|
||||
|
@ -111,12 +111,12 @@ class BandcampBridge extends BridgeAbstract
|
||||
$url = self::URI . 'api/hub/1/dig_deeper';
|
||||
$data = $this->buildRequestJson();
|
||||
$header = [
|
||||
'Content-Type: application/json',
|
||||
'Content-Length: ' . strlen($data)
|
||||
'Content-Type: application/json',
|
||||
'Content-Length: ' . strlen($data),
|
||||
];
|
||||
$opts = [
|
||||
CURLOPT_CUSTOMREQUEST => 'POST',
|
||||
CURLOPT_POSTFIELDS => $data
|
||||
CURLOPT_CUSTOMREQUEST => 'POST',
|
||||
CURLOPT_POSTFIELDS => $data,
|
||||
];
|
||||
$content = getContents($url, $header, $opts);
|
||||
|
||||
@ -314,7 +314,8 @@ class BandcampBridge extends BridgeAbstract
|
||||
{
|
||||
$url = self::URI . 'api/' . $endpoint . '?' . http_build_query($query_data);
|
||||
// todo: 429 Too Many Requests happens a lot
|
||||
$data = json_decode(getContents($url));
|
||||
$response = getContents($url);
|
||||
$data = json_decode($response);
|
||||
return $data;
|
||||
}
|
||||
|
||||
|
@ -93,8 +93,7 @@ class BandcampDailyBridge extends BridgeAbstract
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM($this->getURI())
|
||||
or returnServerError('Could not request: ' . $this->getURI());
|
||||
$html = getSimpleHTMLDOM($this->getURI());
|
||||
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
|
||||
@ -105,8 +104,7 @@ class BandcampDailyBridge extends BridgeAbstract
|
||||
|
||||
$articlePath = $article->find('a.title', 0)->href;
|
||||
|
||||
$articlePageHtml = getSimpleHTMLDOMCached($articlePath, 3600)
|
||||
or returnServerError('Could not request: ' . $articlePath);
|
||||
$articlePageHtml = getSimpleHTMLDOMCached($articlePath, 3600);
|
||||
|
||||
$item['uri'] = $articlePath;
|
||||
$item['title'] = $articlePageHtml->find('article-title', 0)->innertext;
|
||||
|
139
bridges/BazarakiBridge.php
Normal file
139
bridges/BazarakiBridge.php
Normal file
@ -0,0 +1,139 @@
|
||||
<?php
|
||||
|
||||
class BazarakiBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Bazaraki Bridge';
|
||||
const URI = 'https://bazaraki.com';
|
||||
const DESCRIPTION = 'Fetch adverts from Bazaraki, a Cyprus-based classifieds website.';
|
||||
const MAINTAINER = 'danwain';
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'url' => [
|
||||
'name' => 'URL',
|
||||
'type' => 'text',
|
||||
'required' => true,
|
||||
'title' => 'Enter the URL of the Bazaraki page to fetch adverts from.',
|
||||
'exampleValue' => 'https://www.bazaraki.com/real-estate-for-sale/houses/?lat=0&lng=0&radius=100000',
|
||||
],
|
||||
'limit' => [
|
||||
'name' => 'Limit',
|
||||
'type' => 'number',
|
||||
'required' => false,
|
||||
'title' => 'Enter the number of adverts to fetch. (max 50)',
|
||||
'exampleValue' => '10',
|
||||
'defaultValue' => 10,
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = $this->getInput('url');
|
||||
if (! str_starts_with($url, 'https://www.bazaraki.com/')) {
|
||||
throw new \Exception('Nope');
|
||||
}
|
||||
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
|
||||
$i = 0;
|
||||
foreach ($html->find('div.advert') as $element) {
|
||||
$i++;
|
||||
if ($i > $this->getInput('limit') || $i > 50) {
|
||||
break;
|
||||
}
|
||||
|
||||
$item = [];
|
||||
|
||||
$item['uri'] = 'https://www.bazaraki.com' . $element->find('a.advert__content-title', 0)->href;
|
||||
|
||||
# Get the content
|
||||
$advert = getSimpleHTMLDOM($item['uri']);
|
||||
|
||||
$price = trim($advert->find('div.announcement-price__cost', 0)->plaintext);
|
||||
$name = trim($element->find('a.advert__content-title', 0)->plaintext);
|
||||
|
||||
$item['title'] = $name . ' - ' . $price;
|
||||
|
||||
$time = trim($advert->find('span.date-meta', 0)->plaintext);
|
||||
$time = str_replace('Posted: ', '', $time);
|
||||
|
||||
|
||||
$item['content'] = $this->processAdvertContent($advert);
|
||||
$item['timestamp'] = $this->convertRelativeTime($time);
|
||||
$item['author'] = trim($advert->find('div.author-name', 0)->plaintext);
|
||||
$item['uid'] = $advert->find('span.number-announcement', 0)->plaintext;
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the advert content to clean up HTML
|
||||
*
|
||||
* @param simple_html_dom $advert The SimpleHTMLDOM object for the advert page
|
||||
* @return string Processed HTML content
|
||||
*/
|
||||
private function processAdvertContent($advert)
|
||||
{
|
||||
// Get the content sections
|
||||
$header = $advert->find('div.announcement-content-header', 0);
|
||||
$characteristics = $advert->find('div.announcement-characteristics', 0);
|
||||
$description = $advert->find('div.js-description', 0);
|
||||
$images = $advert->find('div.announcement__images', 0);
|
||||
|
||||
// Remove all favorites divs
|
||||
foreach ($advert->find('div.announcement-meta__favorites') as $favorites) {
|
||||
$favorites->outertext = '';
|
||||
}
|
||||
|
||||
// Replace all <a> tags with their text content
|
||||
foreach ($advert->find('a') as $a) {
|
||||
$a->outertext = $a->innertext;
|
||||
}
|
||||
|
||||
// Format the content with section headers and dividers
|
||||
$formattedContent = '';
|
||||
|
||||
// Add header section
|
||||
$formattedContent .= $header->innertext;
|
||||
$formattedContent .= '<hr/>';
|
||||
|
||||
// Add characteristics section with header
|
||||
$formattedContent .= '<h3>Details</h3>';
|
||||
$formattedContent .= $characteristics->innertext;
|
||||
$formattedContent .= '<hr/>';
|
||||
|
||||
// Add description section with header
|
||||
$formattedContent .= '<h3>Description</h3>';
|
||||
$formattedContent .= $description->innertext;
|
||||
$formattedContent .= '<hr/>';
|
||||
|
||||
// Add images section with header
|
||||
$formattedContent .= '<h3>Images</h3>';
|
||||
$formattedContent .= $images->innertext;
|
||||
|
||||
return $formattedContent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert relative time strings like "Yesterday 12:32" to proper timestamps
|
||||
*
|
||||
* @param string $timeString The relative time string from the website
|
||||
* @return string Timestamp in a format compatible with strtotime()
|
||||
*/
|
||||
private function convertRelativeTime($timeString)
|
||||
{
|
||||
if (strpos($timeString, 'Yesterday') !== false) {
|
||||
// Replace "Yesterday" with actual date
|
||||
$time = str_replace('Yesterday', date('Y-m-d', strtotime('-1 day')), $timeString);
|
||||
return date('Y-m-d H:i:s', strtotime($time));
|
||||
} elseif (strpos($timeString, 'Today') !== false) {
|
||||
// Replace "Today" with actual date
|
||||
$time = str_replace('Today', date('Y-m-d'), $timeString);
|
||||
return date('Y-m-d H:i:s', strtotime($time));
|
||||
} else {
|
||||
// For other formats, return as is and let strtotime handle it
|
||||
return $timeString;
|
||||
}
|
||||
}
|
||||
}
|
@ -1,6 +1,6 @@
|
||||
<?php
|
||||
|
||||
class BlizzardNewsBridge extends XPathAbstract
|
||||
class BlizzardNewsBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Blizzard News';
|
||||
const URI = 'https://news.blizzard.com';
|
||||
@ -35,26 +35,73 @@ class BlizzardNewsBridge extends XPathAbstract
|
||||
];
|
||||
const CACHE_TIMEOUT = 3600;
|
||||
|
||||
const XPATH_EXPRESSION_ITEM = '/html/body/div/div[4]/div[2]/div[2]/div/div/section/ol/li/article';
|
||||
const XPATH_EXPRESSION_ITEM_TITLE = './/div/div[2]/h2';
|
||||
const XPATH_EXPRESSION_ITEM_CONTENT = './/div[@class="ArticleListItem-description"]/div[@class="h6"]';
|
||||
const XPATH_EXPRESSION_ITEM_URI = './/a[@class="ArticleLink ArticleLink"]/@href';
|
||||
const XPATH_EXPRESSION_ITEM_AUTHOR = '';
|
||||
const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/time[@class="ArticleListItem-footerTimestamp"]/@timestamp';
|
||||
const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/div[@class="ArticleListItem-image"]/@style';
|
||||
const XPATH_EXPRESSION_ITEM_CATEGORIES = './/div[@class="ArticleListItem-label"]';
|
||||
const SETTING_FIX_ENCODING = true;
|
||||
private const PRODUCT_IDS = [
|
||||
'blt525c436e4a1b0a97',
|
||||
'blt54fbd3787a705054',
|
||||
'blt2031aef34200656d',
|
||||
'blt795c314400d7ded9',
|
||||
'blt5cfc6affa3ca0638',
|
||||
'blt2e50e1521bb84dc6',
|
||||
'blt376fb94931906b6f',
|
||||
'blt81d46fcb05ab8811',
|
||||
'bltede2389c0a8885aa',
|
||||
'blt24859ba8086fb294',
|
||||
'blte27d02816a8ff3e1',
|
||||
'blt2caca37e42f19839',
|
||||
'blt90855744d00cd378',
|
||||
'bltec70ad0ea4fd6d1d',
|
||||
'blt500c1f8b5470bfdb'
|
||||
];
|
||||
|
||||
private const API_PATH = '/api/news/blizzard?';
|
||||
|
||||
/**
|
||||
* Source Web page URL (should provide either HTML or XML content)
|
||||
* @return string
|
||||
*/
|
||||
protected function getSourceUrl()
|
||||
private function getSourceUrl(): string
|
||||
{
|
||||
$locale = $this->getInput('locale');
|
||||
if ('zh-cn' === $locale) {
|
||||
return 'https://cn.news.blizzard.com';
|
||||
$baseUrl = 'https://cn.news.blizzard.com' . self::API_PATH;
|
||||
} else {
|
||||
$baseUrl = 'https://news.blizzard.com/' . $locale . self::API_PATH;
|
||||
}
|
||||
return 'https://news.blizzard.com/' . $locale;
|
||||
return $baseUrl .= http_build_query([
|
||||
'feedCxpProductIds' => self::PRODUCT_IDS
|
||||
]);
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$feedContent = json_decode(getContents($this->getSourceUrl()), true);
|
||||
|
||||
foreach ($feedContent['feed']['contentItems'] as $entry) {
|
||||
$properties = $entry['properties'];
|
||||
|
||||
$item = [];
|
||||
|
||||
$item['title'] = $this->filterChars($properties['title']);
|
||||
$item['content'] = $this->filterChars($properties['summary']);
|
||||
$item['uri'] = $properties['newsUrl'];
|
||||
$item['author'] = $this->filterChars($properties['author']);
|
||||
$item['timestamp'] = strtotime($properties['lastUpdated']);
|
||||
$item['enclosures'] = [$properties['staticAsset']['imageUrl']];
|
||||
$item['categories'] = [$this->filterChars($properties['cxpProduct']['title'])];
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
private function filterChars($content)
|
||||
{
|
||||
return htmlspecialchars($content, ENT_XML1);
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return <<<icon
|
||||
https://dfbmfbnnydoln.cloudfront.net/production/images/favicons/favicon.ba01bb119359d74970b02902472fd82e96b5aba7.ico
|
||||
icon;
|
||||
}
|
||||
}
|
||||
|
670
bridges/BlueskyBridge.php
Normal file
670
bridges/BlueskyBridge.php
Normal file
@ -0,0 +1,670 @@
|
||||
<?php
|
||||
|
||||
class BlueskyBridge extends BridgeAbstract
|
||||
{
|
||||
//Initial PR by [RSSBridge contributors](https://github.com/RSS-Bridge/rss-bridge/issues/4058).
|
||||
//Modified from [©DIYgod and contributors at RSSHub](https://github.com/DIYgod/RSSHub/tree/master/lib/routes/bsky), MIT License';
|
||||
const NAME = 'Bluesky Bridge';
|
||||
const URI = 'https://bsky.app';
|
||||
const DESCRIPTION = 'Fetches posts from Bluesky';
|
||||
const MAINTAINER = 'mruac';
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'data_source' => [
|
||||
'name' => 'Bluesky Data Source',
|
||||
'type' => 'list',
|
||||
'defaultValue' => 'Profile',
|
||||
'values' => [
|
||||
'Profile' => 'getAuthorFeed',
|
||||
],
|
||||
'title' => 'Select the type of data source to fetch from Bluesky.'
|
||||
],
|
||||
'user_id' => [
|
||||
'name' => 'User Handle or DID',
|
||||
'type' => 'text',
|
||||
'required' => true,
|
||||
'exampleValue' => 'did:plc:z72i7hdynmk6r22z27h6tvur',
|
||||
'title' => 'ATProto / Bsky.app handle or DID'
|
||||
],
|
||||
'feed_filter' => [
|
||||
'name' => 'Feed type',
|
||||
'type' => 'list',
|
||||
'defaultValue' => 'posts_and_author_threads',
|
||||
'values' => [
|
||||
'Posts feed' => 'posts_and_author_threads',
|
||||
'All posts and replies' => 'posts_with_replies',
|
||||
'Root posts only' => 'posts_no_replies',
|
||||
'Media only' => 'posts_with_media',
|
||||
]
|
||||
],
|
||||
|
||||
'include_reposts' => [
|
||||
'name' => 'Include Reposts?',
|
||||
'type' => 'checkbox',
|
||||
'defaultValue' => 'checked'
|
||||
],
|
||||
|
||||
'include_reply_context' => [
|
||||
'name' => 'Include Reply context?',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
|
||||
'verbose_title' => [
|
||||
'name' => 'Use verbose feed item titles?',
|
||||
'type' => 'checkbox'
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
private $profile;
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (isset($this->profile)) {
|
||||
if ($this->profile['handle'] === 'handle.invalid') {
|
||||
return sprintf('Bluesky - %s', $this->profile['displayName']);
|
||||
} else {
|
||||
return sprintf('Bluesky - %s (@%s)', $this->profile['displayName'], $this->profile['handle']);
|
||||
}
|
||||
}
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
if (isset($this->profile)) {
|
||||
if ($this->profile['handle'] === 'handle.invalid') {
|
||||
return self::URI . '/profile/' . $this->profile['did'];
|
||||
} else {
|
||||
return self::URI . '/profile/' . $this->profile['handle'];
|
||||
}
|
||||
}
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
if (isset($this->profile)) {
|
||||
return $this->profile['avatar'];
|
||||
}
|
||||
return parent::getIcon();
|
||||
}
|
||||
|
||||
public function getDescription()
|
||||
{
|
||||
if (isset($this->profile)) {
|
||||
return $this->profile['description'];
|
||||
}
|
||||
return parent::getDescription();
|
||||
}
|
||||
|
||||
private function parseExternal($external, $did)
|
||||
{
|
||||
$description = '';
|
||||
$externalUri = $external['uri'];
|
||||
$externalTitle = e($external['title']);
|
||||
$externalDescription = e($external['description']);
|
||||
$thumb = $external['thumb'] ?? null;
|
||||
|
||||
if (preg_match('/http(|s):\/\/media\.tenor\.com/', $externalUri)) {
|
||||
//tenor gif embed
|
||||
$tenorInterstitial = str_replace('media.tenor.com', 'media1.tenor.com/m', $externalUri);
|
||||
$description .= "<figure><a href=\"$tenorInterstitial\"><img src=\"$externalUri\"/></a><figcaption>$externalTitle</figcaption></figure>";
|
||||
} else {
|
||||
//link embed preview
|
||||
$host = parse_url($externalUri)['host'];
|
||||
$thumbDesc = $thumb ? ('<img src="https://cdn.bsky.app/img/feed_thumbnail/plain/' . $did . '/' . $thumb['ref']['$link'] . '@jpeg"/>') : '';
|
||||
$externalDescription = strlen($externalDescription) > 0 ? "<figcaption>($host) $externalDescription</figcaption>" : '';
|
||||
$description .= '<br><blockquote><b><a href="' . $externalUri . '">' . $externalTitle . '</a></b>';
|
||||
$description .= '<figure>' . $thumbDesc . $externalDescription . '</figure></blockquote>';
|
||||
}
|
||||
return $description;
|
||||
}
|
||||
|
||||
private function textToDescription($record)
|
||||
{
|
||||
if (isset($record['value'])) {
|
||||
$record = $record['value'];
|
||||
}
|
||||
$text = $record['text'];
|
||||
$text_copy = $text;
|
||||
$text = nl2br(e($text));
|
||||
if (isset($record['facets'])) {
|
||||
$facets = $record['facets'];
|
||||
foreach ($facets as $facet) {
|
||||
if ($facet['features'][0]['$type'] === 'app.bsky.richtext.facet#link') {
|
||||
$substring = substr($text_copy, $facet['index']['byteStart'], $facet['index']['byteEnd'] - $facet['index']['byteStart']);
|
||||
$text = str_replace($substring, '<a href="' . $facet['features'][0]['uri'] . '">' . $substring . '</a>', $text);
|
||||
}
|
||||
}
|
||||
}
|
||||
return $text;
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$user_id = $this->getInput('user_id');
|
||||
$handle_match = preg_match('/(?:[a-zA-Z]*\.)+([a-zA-Z](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)/', $user_id, $handle_res); //gets the TLD in $handle_match[1]
|
||||
$did_match = preg_match('/did:plc:[a-z2-7]{24}/', $user_id); //https://github.com/did-method-plc/did-method-plc#identifier-syntax
|
||||
$exclude = ['alt', 'arpa', 'example', 'internal', 'invalid', 'local', 'localhost', 'onion']; //https://en.wikipedia.org/wiki/Top-level_domain#Reserved_domains
|
||||
if ($handle_match == true && array_search($handle_res[1], $exclude) == false) {
|
||||
//valid bsky handle
|
||||
$did = $this->resolveHandle($user_id);
|
||||
} elseif ($did_match == true) {
|
||||
//valid DID
|
||||
$did = $user_id;
|
||||
} else {
|
||||
returnClientError('Invalid ATproto handle or DID provided.');
|
||||
}
|
||||
|
||||
$filter = $this->getInput('feed_filter') ?: 'posts_and_author_threads';
|
||||
$replyContext = $this->getInput('include_reply_context');
|
||||
|
||||
$this->profile = $this->getProfile($did);
|
||||
$authorFeed = $this->getAuthorFeed($did, $filter);
|
||||
|
||||
foreach ($authorFeed['feed'] as $post) {
|
||||
$postRecord = $post['post']['record'];
|
||||
|
||||
$item = [];
|
||||
$item['uri'] = self::URI . '/profile/' . $this->fallbackAuthor($post['post']['author'], 'url') . '/post/' . explode('app.bsky.feed.post/', $post['post']['uri'])[1];
|
||||
$item['title'] = $this->getInput('verbose_title') ? $this->generateVerboseTitle($post) : strtok($postRecord['text'], "\n");
|
||||
$item['timestamp'] = strtotime($postRecord['createdAt']);
|
||||
$item['author'] = $this->fallbackAuthor($post['post']['author'], 'display');
|
||||
|
||||
$postAuthorDID = $post['post']['author']['did'];
|
||||
$postAuthorHandle = $post['post']['author']['handle'] !== 'handle.invalid' ? '<i>@' . $post['post']['author']['handle'] . '</i> ' : '';
|
||||
$postDisplayName = $post['post']['author']['displayName'] ?? '';
|
||||
$postDisplayName = e($postDisplayName);
|
||||
$postUri = $item['uri'];
|
||||
|
||||
if (Debug::isEnabled()) {
|
||||
$url = explode('/', $post['post']['uri']);
|
||||
$this->logger->debug('https://bsky.app/profile/' . $url[2] . '/post/' . $url[4]);
|
||||
}
|
||||
|
||||
$description = '';
|
||||
$description .= '<p>';
|
||||
//post
|
||||
$description .= $this->getPostDescription(
|
||||
$postDisplayName,
|
||||
$postAuthorHandle,
|
||||
$postUri,
|
||||
$postRecord,
|
||||
'post'
|
||||
);
|
||||
|
||||
if (isset($postRecord['embed']['$type'])) {
|
||||
//post link embed
|
||||
if ($postRecord['embed']['$type'] === 'app.bsky.embed.external') {
|
||||
$description .= $this->parseExternal($postRecord['embed']['external'], $postAuthorDID);
|
||||
} elseif (
|
||||
$postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$postRecord['embed']['media']['$type'] === 'app.bsky.embed.external'
|
||||
) {
|
||||
$description .= $this->parseExternal($postRecord['embed']['media']['external'], $postAuthorDID);
|
||||
}
|
||||
|
||||
//post images
|
||||
if (
|
||||
$postRecord['embed']['$type'] === 'app.bsky.embed.images' ||
|
||||
(
|
||||
$postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$postRecord['embed']['media']['$type'] === 'app.bsky.embed.images'
|
||||
)
|
||||
) {
|
||||
$images = $post['post']['embed']['images'] ?? $post['post']['embed']['media']['images'];
|
||||
foreach ($images as $image) {
|
||||
$description .= $this->getPostImageDescription($image);
|
||||
}
|
||||
}
|
||||
|
||||
//post video
|
||||
if (
|
||||
$postRecord['embed']['$type'] === 'app.bsky.embed.video' ||
|
||||
(
|
||||
$postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$postRecord['embed']['media']['$type'] === 'app.bsky.embed.video'
|
||||
)
|
||||
) {
|
||||
$description .= $this->getPostVideoDescription(
|
||||
$postRecord['embed']['video'] ?? $postRecord['embed']['media']['video'],
|
||||
$postAuthorDID
|
||||
);
|
||||
}
|
||||
}
|
||||
$description .= '</p>';
|
||||
|
||||
//quote post
|
||||
if (
|
||||
isset($postRecord['embed']) &&
|
||||
(
|
||||
$postRecord['embed']['$type'] === 'app.bsky.embed.record' ||
|
||||
$postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia'
|
||||
) &&
|
||||
isset($post['post']['embed']['record'])
|
||||
) {
|
||||
$description .= '<p>';
|
||||
$quotedRecord = $post['post']['embed']['record']['record'] ?? $post['post']['embed']['record'];
|
||||
|
||||
if (isset($quotedRecord['notFound']) && $quotedRecord['notFound']) { //deleted post
|
||||
$description .= 'Quoted post deleted.';
|
||||
} elseif (isset($quotedRecord['detached']) && $quotedRecord['detached']) { //detached quote
|
||||
$uri_explode = explode('/', $quotedRecord['uri']);
|
||||
$uri_reconstructed = self::URI . '/profile/' . $uri_explode[2] . '/post/' . $uri_explode[4];
|
||||
$description .= '<a href="' . $uri_reconstructed . '">Quoted post detached.</a>';
|
||||
} elseif (isset($quotedRecord['blocked']) && $quotedRecord['blocked']) { //blocked by quote author
|
||||
$description .= 'Author of quoted post has blocked OP.';
|
||||
} elseif (
|
||||
($quotedRecord['$type'] ?? '') === 'app.bsky.feed.defs#generatorView' ||
|
||||
($quotedRecord['$type'] ?? '') === 'app.bsky.graph.defs#listView'
|
||||
) {
|
||||
$description .= $this->getListFeedDescription($quotedRecord);
|
||||
} elseif (
|
||||
($quotedRecord['$type'] ?? '') === 'app.bsky.graph.starterpack' ||
|
||||
($quotedRecord['$type'] ?? '') === 'app.bsky.graph.defs#starterPackViewBasic'
|
||||
) {
|
||||
$description .= $this->getStarterPackDescription($post['post']['embed']['record']);
|
||||
} else {
|
||||
$quotedAuthorDid = $quotedRecord['author']['did'];
|
||||
$quotedDisplayName = $quotedRecord['author']['displayName'] ?? '';
|
||||
$quotedDisplayName = e($quotedDisplayName);
|
||||
$quotedAuthorHandle = $quotedRecord['author']['handle'] !== 'handle.invalid' ? '<i>@' . $quotedRecord['author']['handle'] . '</i>' : '';
|
||||
|
||||
$parts = explode('/', $quotedRecord['uri']);
|
||||
$quotedPostId = end($parts);
|
||||
$quotedPostUri = self::URI . '/profile/' . $this->fallbackAuthor($quotedRecord['author'], 'url') . '/post/' . $quotedPostId;
|
||||
|
||||
//quoted post - post
|
||||
$description .= $this->getPostDescription(
|
||||
$quotedDisplayName,
|
||||
$quotedAuthorHandle,
|
||||
$quotedPostUri,
|
||||
$quotedRecord,
|
||||
'quote'
|
||||
);
|
||||
|
||||
if (isset($quotedRecord['value']['embed']['$type'])) {
|
||||
//quoted post - post link embed
|
||||
if ($quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.external') {
|
||||
$description .= $this->parseExternal($quotedRecord['value']['embed']['external'], $quotedAuthorDid);
|
||||
}
|
||||
|
||||
//quoted post - post video
|
||||
if (
|
||||
$quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.video' ||
|
||||
(
|
||||
$quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$quotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.video'
|
||||
)
|
||||
) {
|
||||
$description .= $this->getPostVideoDescription(
|
||||
$quotedRecord['value']['embed']['video'] ?? $quotedRecord['value']['embed']['media']['video'],
|
||||
$quotedAuthorDid
|
||||
);
|
||||
}
|
||||
|
||||
//quoted post - post images
|
||||
if (
|
||||
$quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.images' ||
|
||||
(
|
||||
$quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$quotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.images'
|
||||
)
|
||||
) {
|
||||
foreach ($quotedRecord['embeds'] as $embed) {
|
||||
if (
|
||||
$embed['$type'] === 'app.bsky.embed.images#view' ||
|
||||
($embed['$type'] === 'app.bsky.embed.recordWithMedia#view' && $embed['media']['$type'] === 'app.bsky.embed.images#view')
|
||||
) {
|
||||
$images = $embed['images'] ?? $embed['media']['images'];
|
||||
foreach ($images as $image) {
|
||||
$description .= $this->getPostImageDescription($image);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$description .= '</p>';
|
||||
}
|
||||
|
||||
//reply
|
||||
if ($replyContext && isset($post['reply']) && !isset($post['reply']['parent']['notFound'])) {
|
||||
$replyPost = $post['reply']['parent'];
|
||||
$replyPostRecord = $replyPost['record'];
|
||||
$description .= '<hr/>';
|
||||
$description .= '<p>';
|
||||
|
||||
$replyPostAuthorDID = $replyPost['author']['did'];
|
||||
$replyPostAuthorHandle = $replyPost['author']['handle'] !== 'handle.invalid' ? '<i>@' . $replyPost['author']['handle'] . '</i> ' : '';
|
||||
$replyPostDisplayName = $replyPost['author']['displayName'] ?? '';
|
||||
$replyPostDisplayName = e($replyPostDisplayName);
|
||||
$replyPostUri = self::URI . '/profile/' . $this->fallbackAuthor($replyPost['author'], 'url') . '/post/' . explode('app.bsky.feed.post/', $replyPost['uri'])[1];
|
||||
|
||||
// reply post
|
||||
$description .= $this->getPostDescription(
|
||||
$replyPostDisplayName,
|
||||
$replyPostAuthorHandle,
|
||||
$replyPostUri,
|
||||
$replyPostRecord,
|
||||
'reply'
|
||||
);
|
||||
|
||||
if (isset($replyPostRecord['embed']['$type'])) {
|
||||
//post link embed
|
||||
if ($replyPostRecord['embed']['$type'] === 'app.bsky.embed.external') {
|
||||
$description .= $this->parseExternal($replyPostRecord['embed']['external'], $replyPostAuthorDID);
|
||||
} elseif (
|
||||
$replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$replyPostRecord['embed']['media']['$type'] === 'app.bsky.embed.external'
|
||||
) {
|
||||
$description .= $this->parseExternal($replyPostRecord['embed']['media']['external'], $replyPostAuthorDID);
|
||||
}
|
||||
|
||||
//post images
|
||||
if (
|
||||
$replyPostRecord['embed']['$type'] === 'app.bsky.embed.images' ||
|
||||
(
|
||||
$replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$replyPostRecord['embed']['media']['$type'] === 'app.bsky.embed.images'
|
||||
)
|
||||
) {
|
||||
$images = $replyPost['embed']['images'] ?? $replyPost['embed']['media']['images'];
|
||||
foreach ($images as $image) {
|
||||
$description .= $this->getPostImageDescription($image);
|
||||
}
|
||||
}
|
||||
|
||||
//post video
|
||||
if (
|
||||
$replyPostRecord['embed']['$type'] === 'app.bsky.embed.video' ||
|
||||
(
|
||||
$replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$replyPostRecord['embed']['media']['$type'] === 'app.bsky.embed.video'
|
||||
)
|
||||
) {
|
||||
$description .= $this->getPostVideoDescription(
|
||||
$replyPostRecord['embed']['video'] ?? $replyPostRecord['embed']['media']['video'],
|
||||
$replyPostAuthorDID
|
||||
);
|
||||
}
|
||||
}
|
||||
$description .= '</p>';
|
||||
|
||||
//quote post
|
||||
if (
|
||||
isset($replyPostRecord['embed']) &&
|
||||
($replyPostRecord['embed']['$type'] === 'app.bsky.embed.record' || $replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia') &&
|
||||
isset($replyPost['embed']['record'])
|
||||
) {
|
||||
$description .= '<p>';
|
||||
$replyQuotedRecord = $replyPost['embed']['record']['record'] ?? $replyPost['embed']['record'];
|
||||
|
||||
if (isset($replyQuotedRecord['notFound']) && $replyQuotedRecord['notFound']) { //deleted post
|
||||
$description .= 'Quoted post deleted.';
|
||||
} elseif (isset($replyQuotedRecord['detached']) && $replyQuotedRecord['detached']) { //detached quote
|
||||
$uri_explode = explode('/', $replyQuotedRecord['uri']);
|
||||
$uri_reconstructed = self::URI . '/profile/' . $uri_explode[2] . '/post/' . $uri_explode[4];
|
||||
$description .= '<a href="' . $uri_reconstructed . '">Quoted post detached.</a>';
|
||||
} elseif (isset($replyQuotedRecord['blocked']) && $replyQuotedRecord['blocked']) { //blocked by quote author
|
||||
$description .= 'Author of quoted post has blocked OP.';
|
||||
} elseif (
|
||||
($replyQuotedRecord['$type'] ?? '') === 'app.bsky.feed.defs#generatorView' ||
|
||||
($replyQuotedRecord['$type'] ?? '') === 'app.bsky.graph.defs#listView'
|
||||
) {
|
||||
$description .= $this->getListFeedDescription($replyQuotedRecord);
|
||||
} elseif (
|
||||
($replyQuotedRecord['$type'] ?? '') === 'app.bsky.graph.starterpack' ||
|
||||
($replyQuotedRecord['$type'] ?? '') === 'app.bsky.graph.defs#starterPackViewBasic'
|
||||
) {
|
||||
$description .= $this->getStarterPackDescription($replyPost['embed']['record']);
|
||||
} else {
|
||||
$quotedAuthorDid = $replyQuotedRecord['author']['did'];
|
||||
$quotedDisplayName = $replyQuotedRecord['author']['displayName'] ?? '';
|
||||
$quotedDisplayName = e($quotedDisplayName);
|
||||
$quotedAuthorHandle = $replyQuotedRecord['author']['handle'] !== 'handle.invalid' ? '<i>@' . $replyQuotedRecord['author']['handle'] . '</i>' : '';
|
||||
|
||||
$parts = explode('/', $replyQuotedRecord['uri']);
|
||||
$quotedPostId = end($parts);
|
||||
$quotedPostUri = self::URI . '/profile/' . $this->fallbackAuthor($replyQuotedRecord['author'], 'url') . '/post/' . $quotedPostId;
|
||||
|
||||
//quoted post - post
|
||||
$description .= $this->getPostDescription(
|
||||
$quotedDisplayName,
|
||||
$quotedAuthorHandle,
|
||||
$quotedPostUri,
|
||||
$replyQuotedRecord,
|
||||
'quote'
|
||||
);
|
||||
|
||||
if (isset($replyQuotedRecord['value']['embed']['$type'])) {
|
||||
//quoted post - post link embed
|
||||
if ($replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.external') {
|
||||
$description .= $this->parseExternal($replyQuotedRecord['value']['embed']['external'], $quotedAuthorDid);
|
||||
}
|
||||
|
||||
//quoted post - post video
|
||||
if (
|
||||
$replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.video' ||
|
||||
(
|
||||
$replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$replyQuotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.video'
|
||||
)
|
||||
) {
|
||||
$description .= $this->getPostVideoDescription(
|
||||
$replyQuotedRecord['value']['embed']['video'] ?? $replyQuotedRecord['value']['embed']['media']['video'],
|
||||
$quotedAuthorDid
|
||||
);
|
||||
}
|
||||
|
||||
//quoted post - post images
|
||||
if (
|
||||
$replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.images' ||
|
||||
(
|
||||
$replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
|
||||
$replyQuotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.images'
|
||||
)
|
||||
) {
|
||||
foreach ($replyQuotedRecord['embeds'] as $embed) {
|
||||
if (
|
||||
$embed['$type'] === 'app.bsky.embed.images#view' ||
|
||||
($embed['$type'] === 'app.bsky.embed.recordWithMedia#view' && $embed['media']['$type'] === 'app.bsky.embed.images#view')
|
||||
) {
|
||||
$images = $embed['images'] ?? $embed['media']['images'];
|
||||
foreach ($images as $image) {
|
||||
$description .= $this->getPostImageDescription($image);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$description .= '</p>';
|
||||
}
|
||||
}
|
||||
|
||||
$item['content'] = $description;
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
private function getPostVideoDescription(array $video, $authorDID)
|
||||
{
|
||||
//https://video.bsky.app/watch/$did/$cid/thumbnail.jpg
|
||||
$videoCID = $video['ref']['$link'];
|
||||
$videoMime = $video['mimeType'];
|
||||
$thumbnail = "poster=\"https://video.bsky.app/watch/$authorDID/$videoCID/thumbnail.jpg\"" ?? '';
|
||||
$videoURL = "https://bsky.social/xrpc/com.atproto.sync.getBlob?did=$authorDID&cid=$videoCID";
|
||||
return "<figure><video loop $thumbnail controls src=\"$videoURL\" type=\"$videoMime\"/></figure>";
|
||||
}
|
||||
|
||||
private function getPostImageDescription(array $image)
|
||||
{
|
||||
$thumbnailUrl = $image['thumb'];
|
||||
$fullsizeUrl = $image['fullsize'];
|
||||
$alt = strlen($image['alt']) > 0 ? '<figcaption>' . e($image['alt']) . '</figcaption>' : '';
|
||||
return "<figure><a href=\"$fullsizeUrl\"><img src=\"$thumbnailUrl\"></a>$alt</figure>";
|
||||
}
|
||||
|
||||
private function getPostDescription(
|
||||
string $postDisplayName,
|
||||
string $postAuthorHandle,
|
||||
string $postUri,
|
||||
array $postRecord,
|
||||
string $type
|
||||
) {
|
||||
$description = '';
|
||||
if ($type === 'quote') {
|
||||
// Quoted post/reply from bbb @bbb.com:
|
||||
$postType = isset($postRecord['reply']) ? 'reply' : 'post';
|
||||
$description .= "<a href=\"$postUri\">Quoted $postType</a> from <b>$postDisplayName</b> $postAuthorHandle:<br>";
|
||||
} elseif ($type === 'reply') {
|
||||
// Replying to aaa @aaa.com's post/reply:
|
||||
$postType = isset($postRecord['reply']) ? 'reply' : 'post';
|
||||
$description .= "Replying to <b>$postDisplayName</b> $postAuthorHandle's <a href=\"$postUri\">$postType</a>:<br>";
|
||||
} else {
|
||||
// aaa @aaa.com posted:
|
||||
$description .= "<b>$postDisplayName</b> $postAuthorHandle <a href=\"$postUri\">posted</a>:<br>";
|
||||
}
|
||||
$description .= $this->textToDescription($postRecord);
|
||||
return $description;
|
||||
}
|
||||
|
||||
//used if handle verification fails, fallsback to displayName or DID depending on context.
|
||||
private function fallbackAuthor($author, $reason)
|
||||
{
|
||||
if ($author['handle'] === 'handle.invalid') {
|
||||
switch ($reason) {
|
||||
case 'url':
|
||||
return $author['did'];
|
||||
case 'display':
|
||||
$displayName = $author['displayName'] ?? '';
|
||||
return e($displayName);
|
||||
}
|
||||
}
|
||||
return $author['handle'];
|
||||
}
|
||||
|
||||
private function generateVerboseTitle($post)
|
||||
{
|
||||
//use "Post by A, replying to B, quoting C" instead of post contents
|
||||
$title = '';
|
||||
if (isset($post['reason']) && str_contains($post['reason']['$type'], 'reasonRepost')) {
|
||||
$title .= 'Repost by ' . $this->fallbackAuthor($post['reason']['by'], 'display') . ', post by ' . $this->fallbackAuthor($post['post']['author'], 'display');
|
||||
} else {
|
||||
$title .= 'Post by ' . $this->fallbackAuthor($post['post']['author'], 'display');
|
||||
}
|
||||
|
||||
if (isset($post['reply'])) {
|
||||
if (isset($post['reply']['parent']['blocked'])) {
|
||||
$replyAuthor = 'blocked user';
|
||||
} elseif (isset($post['reply']['parent']['notFound'])) {
|
||||
$replyAuthor = 'deleted post';
|
||||
} else {
|
||||
$replyAuthor = $this->fallbackAuthor($post['reply']['parent']['author'], 'display');
|
||||
}
|
||||
$title .= ', replying to ' . $replyAuthor;
|
||||
}
|
||||
|
||||
if (
|
||||
isset($post['post']['embed']) &&
|
||||
isset($post['post']['embed']['record']) &&
|
||||
//if not starter pack, feed or list
|
||||
($post['post']['embed']['record']['$type'] ?? '') !== 'app.bsky.feed.defs#generatorView' &&
|
||||
($post['post']['embed']['record']['$type'] ?? '') !== 'app.bsky.graph.defs#listView' &&
|
||||
($post['post']['embed']['record']['$type'] ?? '') !== 'app.bsky.graph.defs#starterPackViewBasic'
|
||||
) {
|
||||
if (isset($post['post']['embed']['record']['blocked'])) {
|
||||
$quotedAuthor = 'blocked user';
|
||||
} elseif (isset($post['post']['embed']['record']['notFound'])) {
|
||||
$quotedAuthor = 'deleted psost';
|
||||
} elseif (isset($post['post']['embed']['record']['detached'])) {
|
||||
$quotedAuthor = 'detached post';
|
||||
} else {
|
||||
$quotedAuthor = $this->fallbackAuthor($post['post']['embed']['record']['record']['author'] ?? $post['post']['embed']['record']['author'], 'display');
|
||||
}
|
||||
$title .= ', quoting ' . $quotedAuthor;
|
||||
}
|
||||
return $title;
|
||||
}
|
||||
|
||||
private function resolveHandle($handle)
|
||||
{
|
||||
$uri = 'https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=' . urlencode($handle);
|
||||
$response = json_decode(getContents($uri), true);
|
||||
return $response['did'];
|
||||
}
|
||||
|
||||
private function getProfile($did)
|
||||
{
|
||||
$uri = 'https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=' . urlencode($did);
|
||||
$response = json_decode(getContents($uri), true);
|
||||
return $response;
|
||||
}
|
||||
|
||||
private function getAuthorFeed($did, $filter)
|
||||
{
|
||||
$uri = 'https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed?actor=' . urlencode($did) . '&filter=' . urlencode($filter) . '&limit=30';
|
||||
if (Debug::isEnabled()) {
|
||||
$this->logger->debug($uri);
|
||||
}
|
||||
$response = json_decode(getContents($uri), true);
|
||||
return $response;
|
||||
}
|
||||
|
||||
//Embed for generated feeds and lists
|
||||
private function getListFeedDescription(array $record): string
|
||||
{
|
||||
$feedViewAvatar = isset($record['avatar']) ? '<img src="' . preg_replace('/\/img\/avatar\//', '/img/avatar_thumbnail/', $record['avatar']) . '">' : '';
|
||||
$feedViewName = e($record['displayName'] ?? $record['name']);
|
||||
$feedViewDescription = e($record['description'] ?? '');
|
||||
$authorDisplayName = e($record['creator']['displayName']);
|
||||
$authorHandle = e($record['creator']['handle']);
|
||||
$likeCount = isset($record['likeCount']) ? '<br>Liked by ' . e($record['likeCount']) . ' users' : '';
|
||||
preg_match('/\/([^\/]+)$/', $record['uri'], $matches);
|
||||
if (($record['purpose'] ?? '') === 'app.bsky.graph.defs#modlist') {
|
||||
$typeURL = '/lists/';
|
||||
$typeDesc = 'moderation list';
|
||||
} elseif (($record['purpose'] ?? '') === 'app.bsky.graph.defs#curatelist') {
|
||||
$typeURL = '/lists/';
|
||||
$typeDesc = 'list';
|
||||
} else {
|
||||
$typeURL = '/feed/';
|
||||
$typeDesc = 'feed';
|
||||
}
|
||||
$uri = e('https://bsky.app/profile/' . $record['creator']['did'] . $typeURL . $matches[1]);
|
||||
|
||||
return <<<END
|
||||
<blockquote>
|
||||
<b><a href="{$uri}">{$feedViewName}</a></b><br/>
|
||||
Bluesky {$typeDesc} by <b>{$authorDisplayName}</b> <i>@{$authorHandle}</i>
|
||||
<figure>
|
||||
{$feedViewAvatar}
|
||||
<figcaption>{$feedViewDescription}{$likeCount}</figcaption>
|
||||
</figure>
|
||||
</blockquote>
|
||||
END;
|
||||
}
|
||||
|
||||
private function getStarterPackDescription(array $record): string
|
||||
{
|
||||
if (!isset($record['record'])) {
|
||||
return 'Failed to get starter pack information.';
|
||||
}
|
||||
$starterpackRecord = $record['record'];
|
||||
$starterpackName = e($starterpackRecord['name']);
|
||||
$starterpackDescription = e($starterpackRecord['description']);
|
||||
$creatorDisplayName = e($record['creator']['displayName']);
|
||||
$creatorHandle = e($record['creator']['handle']);
|
||||
preg_match('/\/([^\/]+)$/', $starterpackRecord['list'], $matches);
|
||||
$uri = e('https://bsky.app/starter-pack/' . $record['creator']['did'] . '/' . $matches[1]);
|
||||
return <<<END
|
||||
<blockquote>
|
||||
<b><a href="{$uri}">{$starterpackName}</a></b><br/>
|
||||
Bluesky starter pack by <b>{$creatorDisplayName}</b> <i>@{$creatorHandle}</i><br/>
|
||||
{$starterpackDescription}
|
||||
</blockquote>
|
||||
END;
|
||||
}
|
||||
}
|
218
bridges/BodaccBridge.php
Normal file
218
bridges/BodaccBridge.php
Normal file
@ -0,0 +1,218 @@
|
||||
<?php
|
||||
|
||||
class BodaccBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'BODACC';
|
||||
const URI = 'https://bodacc-datadila.opendatasoft.com/';
|
||||
const DESCRIPTION = 'Fetches announces from the French Government "Bulletin Officiel Des Annonces Civiles et Commerciales".';
|
||||
const CACHE_TIMEOUT = 86400;
|
||||
const MAINTAINER = 'quent1';
|
||||
const PARAMETERS = [
|
||||
'Annonces commerciales' => [
|
||||
'departement' => [
|
||||
'name' => 'Département',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'Ain' => '01',
|
||||
'Aisne' => '02',
|
||||
'Allier' => '03',
|
||||
'Alpes-de-Haute-Provence' => '04',
|
||||
'Hautes-Alpes' => '05',
|
||||
'Alpes-Maritimes' => '06',
|
||||
'Ardèche' => '07',
|
||||
'Ardennes' => '08',
|
||||
'Ariège' => '09',
|
||||
'Aube' => '10',
|
||||
'Aude' => '11',
|
||||
'Aveyron' => '12',
|
||||
'Bouches-du-Rhône' => '13',
|
||||
'Calvados' => '14',
|
||||
'Cantal' => '15',
|
||||
'Charente' => '16',
|
||||
'Charente-Maritime' => '17',
|
||||
'Cher' => '18',
|
||||
'Corrèze' => '19',
|
||||
'Corse-du-Sud' => '2A',
|
||||
'Haute-Corse' => '2B',
|
||||
'Côte-d\'Or' => '21',
|
||||
'Côtes-d\'Armor' => '22',
|
||||
'Creuse' => '23',
|
||||
'Dordogne' => '24',
|
||||
'Doubs' => '25',
|
||||
'Drôme' => '26',
|
||||
'Eure' => '27',
|
||||
'Eure-et-Loir' => '28',
|
||||
'Finistère' => '29',
|
||||
'Gard' => '30',
|
||||
'Haute-Garonne' => '31',
|
||||
'Gers' => '32',
|
||||
'Gironde' => '33',
|
||||
'Hérault' => '34',
|
||||
'Ille-et-Vilaine' => '35',
|
||||
'Indre' => '36',
|
||||
'Indre-et-Loire' => '37',
|
||||
'Isère' => '38',
|
||||
'Jura' => '39',
|
||||
'Landes' => '40',
|
||||
'Loir-et-Cher' => '41',
|
||||
'Loire' => '42',
|
||||
'Haute-Loire' => '43',
|
||||
'Loire-Atlantique' => '44',
|
||||
'Loiret' => '45',
|
||||
'Lot' => '46',
|
||||
'Lot-et-Garonne' => '47',
|
||||
'Lozère' => '48',
|
||||
'Maine-et-Loire' => '49',
|
||||
'Manche' => '50',
|
||||
'Marne' => '51',
|
||||
'Haute-Marne' => '52',
|
||||
'Mayenne' => '53',
|
||||
'Meurthe-et-Moselle' => '54',
|
||||
'Meuse' => '55',
|
||||
'Morbihan' => '56',
|
||||
'Moselle' => '57',
|
||||
'Nièvre' => '58',
|
||||
'Nord' => '59',
|
||||
'Oise' => '60',
|
||||
'Orne' => '61',
|
||||
'Pas-de-Calais' => '62',
|
||||
'Puy-de-Dôme' => '63',
|
||||
'Pyrénées-Atlantiques' => '64',
|
||||
'Hautes-Pyrénées' => '65',
|
||||
'Pyrénées-Orientales' => '66',
|
||||
'Bas-Rhin' => '67',
|
||||
'Haut-Rhin' => '68',
|
||||
'Rhône' => '69',
|
||||
'Haute-Saône' => '70',
|
||||
'Saône-et-Loire' => '71',
|
||||
'Sarthe' => '72',
|
||||
'Savoie' => '73',
|
||||
'Haute-Savoie' => '74',
|
||||
'Paris' => '75',
|
||||
'Seine-Maritime' => '76',
|
||||
'Seine-et-Marne' => '77',
|
||||
'Yvelines' => '78',
|
||||
'Deux-Sèvres' => '79',
|
||||
'Somme' => '80',
|
||||
'Tarn' => '81',
|
||||
'Tarn-et-Garonne' => '82',
|
||||
'Var' => '83',
|
||||
'Vaucluse' => '84',
|
||||
'Vendée' => '85',
|
||||
'Vienne' => '86',
|
||||
'Haute-Vienne' => '87',
|
||||
'Vosges' => '88',
|
||||
'Yonne' => '89',
|
||||
'Territoire de Belfort' => '90',
|
||||
'Essonne' => '91',
|
||||
'Hauts-de-Seine' => '92',
|
||||
'Seine-Saint-Denis' => '93',
|
||||
'Val-de-Marne' => '94',
|
||||
'Val-d\'Oise' => '95',
|
||||
'Guadeloupe' => '971',
|
||||
'Martinique' => '972',
|
||||
'Guyane' => '973',
|
||||
'La Réunion' => '974',
|
||||
'Saint-Pierre-et-Miquelon' => '975',
|
||||
'Mayotte' => '976',
|
||||
'Saint-Barthélemy' => '977',
|
||||
'Saint-Martin' => '978',
|
||||
'Terres australes et antarctiques françaises' => '984',
|
||||
'Wallis-et-Futuna' => '986',
|
||||
'Polynésie française' => '987',
|
||||
'Nouvelle-Calédonie' => '988',
|
||||
'Île de Clipperton' => '989'
|
||||
]
|
||||
],
|
||||
'famille' => [
|
||||
'name' => 'Famille',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Toutes' => null,
|
||||
'Annonces diverses' => 'divers',
|
||||
'Créations' => 'creation',
|
||||
'Dépôts des comptes' => 'dpc',
|
||||
'Immatriculations' => 'immatriculation',
|
||||
'Modifications diverses' => 'modification',
|
||||
'Procédures collectives' => 'collective',
|
||||
'Procédures de conciliation' => 'conciliation',
|
||||
'Procédures de rétablissement professionnel' => 'retablissement_professionnel',
|
||||
'Radiations' => 'radiation',
|
||||
'Ventes et cessions' => 'vente'
|
||||
]
|
||||
],
|
||||
'type' => [
|
||||
'name' => 'Type',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'Avis initial' => 'annonce',
|
||||
'Avis d\'annulation' => 'annulation',
|
||||
'Avis rectificatif' => 'rectificatif'
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$parameters = [
|
||||
'select' => 'id,dateparution,typeavis_lib,familleavis_lib,commercant,ville,cp',
|
||||
'order_by' => 'id desc',
|
||||
'limit' => 50,
|
||||
];
|
||||
|
||||
$where = [];
|
||||
if (!empty($this->getInput('departement'))) {
|
||||
$where[] = 'numerodepartement="' . $this->getInput('departement') . '"';
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('famille'))) {
|
||||
$where[] = 'familleavis="' . $this->getInput('famille') . '"';
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('type'))) {
|
||||
$where[] = 'typeavis="' . $this->getInput('type') . '"';
|
||||
}
|
||||
|
||||
if ($where !== []) {
|
||||
$parameters['where'] = implode(' and ', $where);
|
||||
}
|
||||
|
||||
$url = urljoin(self::URI, '/api/explore/v2.1/catalog/datasets/annonces-commerciales/records?' . http_build_query($parameters));
|
||||
|
||||
$data = Json::decode(getContents($url), false);
|
||||
|
||||
foreach ($data->results as $result) {
|
||||
if (
|
||||
!isset(
|
||||
$result->id,
|
||||
$result->dateparution,
|
||||
$result->typeavis_lib,
|
||||
$result->familleavis_lib,
|
||||
$result->commercant,
|
||||
$result->ville,
|
||||
$result->cp
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$title = sprintf(
|
||||
'[%s] %s - %s à %s (%s)',
|
||||
$result->typeavis_lib,
|
||||
$result->familleavis_lib,
|
||||
$result->commercant,
|
||||
$result->ville,
|
||||
$result->cp
|
||||
);
|
||||
|
||||
$this->items[] = [
|
||||
'uid' => $result->id,
|
||||
'timestamp' => strtotime($result->dateparution),
|
||||
'title' => $title,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
@ -1218,14 +1218,15 @@ EOT;
|
||||
$table = $this->generateEventDetailsTable($event);
|
||||
|
||||
$imgsrc = $event['BannerURL'];
|
||||
$FShareURL = $event['FShareURL'];
|
||||
|
||||
return <<<EOT
|
||||
<img title="Event Banner URL" src="$imgsrc"></img>
|
||||
<br>
|
||||
$table
|
||||
<br>
|
||||
More Details are available on the <a href="${event['FShareURL']}">BookMyShow website</a>.
|
||||
EOT;
|
||||
<img title="Event Banner URL" src="$imgsrc">
|
||||
<br>
|
||||
$table
|
||||
<br>
|
||||
More Details are available on the <a href="$FShareURL">BookMyShow website</a>.
|
||||
EOT;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1292,14 +1293,15 @@ EOT;
|
||||
|
||||
$synopsis = preg_replace(self::SYNOPSIS_REGEX, '', $data['EventSynopsis']);
|
||||
|
||||
$eventTrailerURL = $data['EventTrailerURL'];
|
||||
return <<<EOT
|
||||
<img title="Movie Poster" src="$imgsrc"></img>
|
||||
<div>$table</div>
|
||||
<p>$innerHtml</p>
|
||||
<p>${synopsis}</p>
|
||||
More Details are available on the <a href="$url">BookMyShow website</a> and a trailer is available
|
||||
<a href="${data['EventTrailerURL']}" title="Trailer URL">here</a>
|
||||
EOT;
|
||||
<img title="Movie Poster" src="$imgsrc"></img>
|
||||
<div>$table</div>
|
||||
<p>$innerHtml</p>
|
||||
<p>$synopsis</p>
|
||||
More Details are available on the <a href="$url">BookMyShow website</a> and a trailer is available
|
||||
<a href="$eventTrailerURL" title="Trailer URL">here</a>
|
||||
EOT;
|
||||
}
|
||||
|
||||
/**
|
||||
|
63
bridges/BruegelBridge.php
Normal file
63
bridges/BruegelBridge.php
Normal file
@ -0,0 +1,63 @@
|
||||
<?php
|
||||
|
||||
class BruegelBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Bruegel';
|
||||
const URI = 'https://www.bruegel.org';
|
||||
const DESCRIPTION = 'European think-tank commentary and publications.';
|
||||
const MAINTAINER = 'KappaPrajd';
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'category' => [
|
||||
'name' => 'Category',
|
||||
'type' => 'list',
|
||||
'defaultValue' => '/publications',
|
||||
'values' => [
|
||||
'Publications' => '/publications',
|
||||
'Commentary' => '/commentary'
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return self::URI . '/themes/custom/bruegel/assets/favicon/android-icon-72x72.png';
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = self::URI . $this->getInput('category');
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
|
||||
$articles = $html->find('.c-listing__content article');
|
||||
|
||||
foreach ($articles as $article) {
|
||||
$title = $article->find('.c-list-item__title a span', 0)->plaintext;
|
||||
$content = trim($article->find('.c-list-item__description', 0)->plaintext);
|
||||
$publishDate = $article->find('.c-list-item__date', 0)->plaintext;
|
||||
$href = $article->find('.c-list-item__title a', 0)->getAttribute('href');
|
||||
|
||||
$item = [
|
||||
'title' => $title,
|
||||
'content' => $content,
|
||||
'timestamp' => strtotime($publishDate),
|
||||
'uri' => self::URI . $href,
|
||||
'author' => $this->getAuthor($article),
|
||||
];
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
private function getAuthor($article)
|
||||
{
|
||||
$authorsElements = $article->find('.c-list-item__authors a');
|
||||
|
||||
$authors = array_map(function ($author) {
|
||||
return $author->plaintext;
|
||||
}, $authorsElements);
|
||||
|
||||
return join(', ', $authors);
|
||||
}
|
||||
}
|
@ -164,7 +164,7 @@ class BugzillaBridge extends BridgeAbstract
|
||||
}
|
||||
|
||||
$cache = $this->loadCacheValue($this->instance . $user);
|
||||
if (!is_null($cache)) {
|
||||
if ($cache) {
|
||||
return $cache;
|
||||
}
|
||||
|
||||
|
@ -206,7 +206,7 @@ class BukowskisBridge extends BridgeAbstract
|
||||
$this->items[] = [
|
||||
'title' => $title,
|
||||
'uri' => $baseUrl . $relative_url,
|
||||
'uid' => $lot->getAttribute('data-lot-id'),
|
||||
'uid' => $relative_url,
|
||||
'content' => count($images) > 0 ? "<img src='$images[0]'/><br/>$title" : $title,
|
||||
'enclosures' => array_slice($images, 1),
|
||||
];
|
||||
|
@ -26,18 +26,16 @@ TMPL;
|
||||
https://www.bundestag.de/ajax/filterlist/de/parlament/praesidium/parteienfinanzierung/fundstellen50000/462002-462002
|
||||
URI;
|
||||
// Get the main page
|
||||
$html = getSimpleHTMLDOMCached($ajaxUri, self::CACHE_TIMEOUT)
|
||||
or returnServerError('Could not request AJAX list.');
|
||||
$html = getSimpleHTMLDOMCached($ajaxUri, self::CACHE_TIMEOUT);
|
||||
|
||||
// Build the URL from the first anchor element. The list is sorted by year, descending, so the first element is the current year.
|
||||
$firstAnchor = $html->find('a', 0)
|
||||
or returnServerError('Could not find the proper HTML element.');
|
||||
|
||||
$url = 'https://www.bundestag.de' . $firstAnchor->href;
|
||||
$url = $firstAnchor->href;
|
||||
|
||||
// Get the actual page with the soft money donations
|
||||
$html = getSimpleHTMLDOMCached($url, self::CACHE_TIMEOUT)
|
||||
or returnServerError('Could not request ' . $url);
|
||||
$html = getSimpleHTMLDOMCached($url, self::CACHE_TIMEOUT);
|
||||
|
||||
$rows = $html->find('table.table > tbody > tr')
|
||||
or returnServerError('Could not find the proper HTML elements.');
|
||||
|
@ -56,7 +56,7 @@ class CNETBridge extends SitemapBridge
|
||||
foreach ($links as $article_uri) {
|
||||
$article_dom = convertLazyLoading(getSimpleHTMLDOMCached($article_uri));
|
||||
$title = trim($article_dom->find('h1', 0)->plaintext);
|
||||
$author = $article_dom->find('span.c-assetAuthor_name', 0)->plaintext;
|
||||
$author = $article_dom->find('span.c-assetAuthor_name', 0);
|
||||
$headline = $article_dom->find('p.c-contentHeader_description', 0);
|
||||
$content = $article_dom->find('div.c-pageArticle_content, div.single-article__content, div.article-main-body', 0);
|
||||
$date = null;
|
||||
@ -97,7 +97,11 @@ class CNETBridge extends SitemapBridge
|
||||
$item = [];
|
||||
$item['uri'] = $article_uri;
|
||||
$item['title'] = $title;
|
||||
$item['author'] = $author;
|
||||
|
||||
if ($author) {
|
||||
$item['author'] = $author->plaintext;
|
||||
}
|
||||
|
||||
$item['content'] = $content;
|
||||
|
||||
if (!is_null($date)) {
|
||||
|
@ -42,45 +42,23 @@ class CVEDetailsBridge extends BridgeAbstract
|
||||
$this->fetchContent();
|
||||
}
|
||||
|
||||
foreach ($this->html->find('#searchresults > .row') as $i => $tr) {
|
||||
// There are some optional vulnerability types, which will be
|
||||
// added to the categories as well as the CWE number -- which is
|
||||
// always given.
|
||||
$categories = [$this->vendor];
|
||||
$enclosures = [];
|
||||
|
||||
$detailLink = $tr->find('h3 > a', 0);
|
||||
$detailHtml = getSimpleHTMLDOM($detailLink->href);
|
||||
|
||||
// The CVE number itself
|
||||
$var = $this->html->find('#searchresults > div > div.row');
|
||||
foreach ($var as $i => $tr) {
|
||||
$uri = $tr->find('h3 > a', 0)->href ?? null;
|
||||
$title = $tr->find('h3 > a', 0)->innertext;
|
||||
$content = $tr->find('.cvesummarylong', 0)->innertext;
|
||||
$cweList = $detailHtml->find('h2', 2)->next_sibling();
|
||||
foreach ($cweList->find('li') as $li) {
|
||||
$cweWithDescription = $li->find('a', 0)->innertext ?? '';
|
||||
|
||||
if (preg_match('/CWE-(\d+)/', $cweWithDescription, $cwe)) {
|
||||
$categories[] = 'CWE-' . $cwe[1];
|
||||
$enclosures[] = 'https://cwe.mitre.org/data/definitions/' . $cwe[1] . '.html';
|
||||
}
|
||||
}
|
||||
|
||||
if ($this->product != '') {
|
||||
$categories[] = $this->product;
|
||||
}
|
||||
$content = $tr->find('.cvesummarylong', 0)->innertext ?? '';
|
||||
$timestamp = $tr->find('[data-tsvfield="publishDate"]', 0)->innertext ?? 0;
|
||||
|
||||
$this->items[] = [
|
||||
'uri' => 'https://cvedetails.com/' . $detailHtml->find('h1 > a', 0)->href,
|
||||
'uri' => $uri,
|
||||
'title' => $title,
|
||||
'timestamp' => $tr->find('[data-tsvfield="publishDate"]', 0)->innertext,
|
||||
'timestamp' => $timestamp,
|
||||
'content' => $content,
|
||||
'categories' => $categories,
|
||||
'enclosures' => $enclosures,
|
||||
'categories' => [$this->vendor],
|
||||
'enclosures' => [],
|
||||
'uid' => $title,
|
||||
];
|
||||
|
||||
// We only want to fetch the latest 10 CVEs
|
||||
if (count($this->items) >= 10) {
|
||||
if (count($this->items) >= 30) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
@ -6,46 +6,113 @@ class CarThrottleBridge extends BridgeAbstract
|
||||
const URI = 'https://www.carthrottle.com/';
|
||||
const DESCRIPTION = 'Get the latest car-related news from Car Throttle.';
|
||||
const MAINTAINER = 't0stiman';
|
||||
const DONATION_URI = 'https://ko-fi.com/tostiman';
|
||||
|
||||
const PARAMETERS = [
|
||||
'Show articles from these categories:' => [
|
||||
'news' => [
|
||||
'name' => 'news',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'reviews' => [
|
||||
'name' => 'reviews',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'features' => [
|
||||
'name' => 'features',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'videos' => [
|
||||
'name' => 'videos',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'gaming' => [
|
||||
'name' => 'gaming',
|
||||
'type' => 'checkbox'
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$news = getSimpleHTMLDOMCached(self::URI . 'news');
|
||||
$this->items = [];
|
||||
|
||||
$this->items[] = [];
|
||||
$this->handleCategory('news');
|
||||
$this->handleCategory('reviews');
|
||||
$this->handleCategory('features');
|
||||
$this->handleCategory2('videos', 'video');
|
||||
$this->handleCategory('gaming');
|
||||
}
|
||||
|
||||
private function handleCategory($category)
|
||||
{
|
||||
if ($this->getInput($category)) {
|
||||
$this->getArticles($category);
|
||||
}
|
||||
}
|
||||
|
||||
private function handleCategory2($categoryParameter, $categoryURLname)
|
||||
{
|
||||
if ($this->getInput($categoryParameter)) {
|
||||
$this->getArticles($categoryURLname);
|
||||
}
|
||||
}
|
||||
|
||||
private function getArticles($category)
|
||||
{
|
||||
$categoryPage = getSimpleHTMLDOMCached(self::URI . $category);
|
||||
|
||||
//for each post
|
||||
foreach ($news->find('div.cmg-card') as $post) {
|
||||
foreach ($categoryPage->find('div.cmg-card') as $post) {
|
||||
$item = [];
|
||||
|
||||
$titleElement = $post->find('div.title a.cmg-link')[0];
|
||||
$item['uri'] = self::URI . $titleElement->getAttribute('href');
|
||||
$titleElement = $post->find('a.title')[0];
|
||||
$post_uri = self::URI . $titleElement->getAttribute('href');
|
||||
|
||||
if (!isset($post_uri) || $post_uri == '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$item['uri'] = $post_uri;
|
||||
$item['title'] = $titleElement->innertext;
|
||||
|
||||
$articlePage = getSimpleHTMLDOMCached($item['uri']);
|
||||
|
||||
$authorDiv = $articlePage->find('div.author div');
|
||||
if ($authorDiv) {
|
||||
$item['author'] = $authorDiv[1]->innertext;
|
||||
}
|
||||
$item['author'] = $this->parseAuthor($articlePage);
|
||||
|
||||
$articleImage = $articlePage->find('figure')[0];
|
||||
$article = $articlePage->find('div.first-column div.body')[0];
|
||||
|
||||
$dinges = $articlePage->find('div.main-body')[0] ?? null;
|
||||
//remove ads
|
||||
if ($dinges) {
|
||||
foreach ($dinges->find('aside') as $ad) {
|
||||
$ad->outertext = '';
|
||||
$dinges->save();
|
||||
}
|
||||
foreach ($article->find('aside') as $ad) {
|
||||
$ad->outertext = '';
|
||||
}
|
||||
|
||||
$var = $articlePage->find('div.summary')[0] ?? '';
|
||||
$var1 = $articlePage->find('figure.main-image')[0] ?? '';
|
||||
$dinges1 = $dinges ?? '';
|
||||
$summary = $articlePage->find('div.summary')[0];
|
||||
|
||||
$item['content'] = $var .
|
||||
$var1 .
|
||||
$dinges1;
|
||||
//these are supposed to be hidden
|
||||
foreach ($article->find('.visually-hidden') as $found) {
|
||||
$found->outertext = '';
|
||||
}
|
||||
|
||||
$item['content'] = $summary . $articleImage . $article;
|
||||
|
||||
array_push($this->items, $item);
|
||||
}
|
||||
}
|
||||
|
||||
private function parseAuthor($articlePage)
|
||||
{
|
||||
$authorDivs = $articlePage->find('div address');
|
||||
if (!$authorDivs) {
|
||||
return '';
|
||||
}
|
||||
|
||||
$a = $authorDivs[0]->find('a')[0];
|
||||
if ($a) {
|
||||
return $a->innertext;
|
||||
}
|
||||
|
||||
return $authorDivs[0]->innertext;
|
||||
}
|
||||
}
|
||||
|
@ -54,7 +54,7 @@ class CaschyBridge extends FeedExpander
|
||||
{
|
||||
// remove unwanted stuff
|
||||
foreach (
|
||||
$article->find('div.video-container, div.aawp, p.aawp-disclaimer, iframe.wp-embedded-content,
|
||||
$article->find('div.aawp, p.aawp-disclaimer, iframe.wp-embedded-content,
|
||||
div.wp-embed, p.wp-caption-text, script') as $element
|
||||
) {
|
||||
$element->remove();
|
||||
|
266
bridges/CentreFranceBridge.php
Normal file
266
bridges/CentreFranceBridge.php
Normal file
@ -0,0 +1,266 @@
|
||||
<?php
|
||||
|
||||
class CentreFranceBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Centre France Newspapers';
|
||||
const URI = 'https://www.centrefrance.com/';
|
||||
const DESCRIPTION = 'Common bridge for all Centre France group newspapers.';
|
||||
const CACHE_TIMEOUT = 7200; // 2h
|
||||
const MAINTAINER = 'quent1';
|
||||
const PARAMETERS = [
|
||||
'global' => [
|
||||
'newspaper' => [
|
||||
'name' => 'Newspaper',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'La Montagne' => 'lamontagne.fr',
|
||||
'Le Populaire du Centre' => 'lepopulaire.fr',
|
||||
'La République du Centre' => 'larep.fr',
|
||||
'Le Berry Républicain' => 'leberry.fr',
|
||||
'L\'Yonne Républicaine' => 'lyonne.fr',
|
||||
'L\'Écho Républicain' => 'lechorepublicain.fr',
|
||||
'Le Journal du Centre' => 'lejdc.fr',
|
||||
'L\'Éveil de la Haute-Loire' => 'leveil.fr',
|
||||
'Le Pays' => 'le-pays.fr'
|
||||
]
|
||||
],
|
||||
'remove-reserved-for-subscribers-articles' => [
|
||||
'name' => 'Remove reserved for subscribers articles',
|
||||
'type' => 'checkbox',
|
||||
'title' => 'Filter out articles that are only available to subscribers'
|
||||
],
|
||||
'limit' => [
|
||||
'name' => 'Limit',
|
||||
'type' => 'number',
|
||||
'title' => 'How many articles to fetch. 0 to disable.',
|
||||
'required' => true,
|
||||
'defaultValue' => 15
|
||||
]
|
||||
],
|
||||
'Local news' => [
|
||||
'locality-slug' => [
|
||||
'name' => 'Locality slug',
|
||||
'type' => 'text',
|
||||
'required' => false,
|
||||
'title' => 'Fetch articles for a specific locality. If not set, headlines from the front page will be used instead.',
|
||||
'exampleValue' => 'moulins-03000'
|
||||
],
|
||||
]
|
||||
];
|
||||
|
||||
private static array $monthNumberByFrenchName = [
|
||||
'janvier' => 1, 'février' => 2, 'mars' => 3, 'avril' => 4, 'mai' => 5, 'juin' => 6, 'juillet' => 7,
|
||||
'août' => 8, 'septembre' => 9, 'octobre' => 10, 'novembre' => 11, 'décembre' => 12
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$value = $this->getInput('limit');
|
||||
if (is_numeric($value) && (int)$value >= 0) {
|
||||
$limit = $value;
|
||||
} else {
|
||||
$limit = static::PARAMETERS['global']['limit']['defaultValue'];
|
||||
}
|
||||
|
||||
if (empty($this->getInput('newspaper'))) {
|
||||
return;
|
||||
}
|
||||
|
||||
$localitySlug = $this->getInput('locality-slug') ?? '';
|
||||
$alreadyFoundArticlesURIs = [];
|
||||
|
||||
$newspaperUrl = 'https://www.' . $this->getInput('newspaper') . '/' . $localitySlug . '/';
|
||||
$html = getSimpleHTMLDOM($newspaperUrl);
|
||||
|
||||
// Articles are detected through their titles
|
||||
foreach ($html->find('.c-titre') as $articleTitleDOMElement) {
|
||||
$articleLinkDOMElement = $articleTitleDOMElement->find('a', 0);
|
||||
|
||||
// Ignore articles in the « Les + partagés » block
|
||||
if (strpos($articleLinkDOMElement->id, 'les_plus_partages') !== false) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$articleURI = $articleLinkDOMElement->href;
|
||||
|
||||
// If the URI has already been processed, ignore it
|
||||
if (in_array($articleURI, $alreadyFoundArticlesURIs, true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If news are filtered for a specific locality, filter out article for other localities
|
||||
if ($localitySlug !== '' && !str_contains($articleURI, $localitySlug)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$articleTitle = '';
|
||||
|
||||
// If article is reserved for subscribers
|
||||
if ($articleLinkDOMElement->find('span.premium-picto', 0)) {
|
||||
if ($this->getInput('remove-reserved-for-subscribers-articles') === true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$articleTitle .= '🔒 ';
|
||||
}
|
||||
|
||||
$articleTitleDOMElement = $articleLinkDOMElement->find('span[data-tb-title]', 0);
|
||||
if ($articleTitleDOMElement === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($limit > 0 && count($this->items) === $limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
$articleTitle .= $articleLinkDOMElement->find('span[data-tb-title]', 0)->innertext;
|
||||
$articleFullURI = urljoin('https://www.' . $this->getInput('newspaper') . '/', $articleURI);
|
||||
|
||||
$item = [
|
||||
'title' => $articleTitle,
|
||||
'uri' => $articleFullURI,
|
||||
...$this->collectArticleData($articleFullURI)
|
||||
];
|
||||
$this->items[] = $item;
|
||||
|
||||
$alreadyFoundArticlesURIs[] = $articleURI;
|
||||
}
|
||||
}
|
||||
|
||||
private function collectArticleData($uri): array
|
||||
{
|
||||
$html = getSimpleHTMLDOMCached($uri, 86400 * 90); // 90d
|
||||
|
||||
$item = [
|
||||
'enclosures' => [],
|
||||
];
|
||||
|
||||
$articleInformations = $html->find('#content hgroup > div.typo-p3 > *');
|
||||
if (is_array($articleInformations) && $articleInformations !== []) {
|
||||
$publicationDateIndex = 0;
|
||||
|
||||
// Article author
|
||||
$probableAuthorName = strip_tags($articleInformations[0]->innertext);
|
||||
if (str_starts_with($probableAuthorName, 'Par ')) {
|
||||
$publicationDateIndex = 1;
|
||||
$item['author'] = substr($probableAuthorName, 4);
|
||||
}
|
||||
|
||||
// Article publication date
|
||||
preg_match('/Publié le (\d{2}) (.+) (\d{4})( à (\d{2})h(\d{2}))?/', strip_tags($articleInformations[$publicationDateIndex]->innertext), $articleDateParts);
|
||||
if ($articleDateParts !== [] && array_key_exists($articleDateParts[2], self::$monthNumberByFrenchName)) {
|
||||
$articleDate = new \DateTime('midnight');
|
||||
$articleDate->setDate($articleDateParts[3], self::$monthNumberByFrenchName[$articleDateParts[2]], $articleDateParts[1]);
|
||||
|
||||
if (count($articleDateParts) === 7) {
|
||||
$articleDate->setTime($articleDateParts[5], $articleDateParts[6]);
|
||||
}
|
||||
|
||||
$item['timestamp'] = $articleDate->getTimestamp();
|
||||
}
|
||||
}
|
||||
|
||||
$articleContent = $html->find('#content>div.flex+div.grid section>.z-10')[0] ?? null;
|
||||
if ($articleContent instanceof \simple_html_dom_node) {
|
||||
$articleHiddenParts = $articleContent->find('.ad-slot, #cf-digiteka-player');
|
||||
if (is_array($articleHiddenParts)) {
|
||||
foreach ($articleHiddenParts as $articleHiddenPart) {
|
||||
$articleContent->removeChild($articleHiddenPart);
|
||||
}
|
||||
}
|
||||
|
||||
$item['content'] = $articleContent->innertext;
|
||||
}
|
||||
|
||||
$articleIllustration = $html->find('#content>div.flex+div.grid section>figure>img');
|
||||
if (is_array($articleIllustration) && count($articleIllustration) === 1) {
|
||||
$item['enclosures'][] = $articleIllustration[0]->getAttribute('src');
|
||||
}
|
||||
|
||||
$articleAudio = $html->find('audio[src^="https://api.octopus.saooti.com/"]');
|
||||
if (is_array($articleAudio) && count($articleAudio) === 1) {
|
||||
$item['enclosures'][] = $articleAudio[0]->getAttribute('src');
|
||||
}
|
||||
|
||||
$articleTags = $html->find('#content>div.flex+div.grid section>.bg-gray-light>a.border-gray-dark');
|
||||
if (is_array($articleTags)) {
|
||||
$item['categories'] = array_map(static fn ($articleTag) => $articleTag->innertext, $articleTags);
|
||||
}
|
||||
|
||||
$explode = explode('_', $uri);
|
||||
$array_reverse = array_reverse($explode);
|
||||
$string = $array_reverse[0];
|
||||
$uid = rtrim($string, '/');
|
||||
if (is_numeric($uid)) {
|
||||
$item['uid'] = $uid;
|
||||
}
|
||||
|
||||
// If the article is a "grand format", we use another parsing strategy
|
||||
if ($item['content'] === '' && $html->find('article') !== []) {
|
||||
$articleContent = $html->find('article > section');
|
||||
foreach ($articleContent as $contentPart) {
|
||||
if ($contentPart->find('#journo') !== []) {
|
||||
$item['author'] = $contentPart->find('#journo')->innertext;
|
||||
continue;
|
||||
}
|
||||
|
||||
$item['content'] .= $contentPart->innertext;
|
||||
}
|
||||
}
|
||||
|
||||
$item['content'] = str_replace('<span class="p-premium">premium</span>', '🔒', $item['content']);
|
||||
$item['content'] = trim($item['content']);
|
||||
|
||||
return $item;
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (empty($this->getInput('newspaper'))) {
|
||||
return static::NAME;
|
||||
}
|
||||
|
||||
$newspaperNameByDomain = array_flip(self::PARAMETERS['global']['newspaper']['values']);
|
||||
if (!isset($newspaperNameByDomain[$this->getInput('newspaper')])) {
|
||||
return static::NAME;
|
||||
}
|
||||
|
||||
$completeTitle = $newspaperNameByDomain[$this->getInput('newspaper')];
|
||||
|
||||
if (!empty($this->getInput('locality-slug'))) {
|
||||
$localityName = explode('-', $this->getInput('locality-slug'));
|
||||
array_pop($localityName);
|
||||
$completeTitle .= ' ' . ucfirst(implode('-', $localityName));
|
||||
}
|
||||
|
||||
return $completeTitle;
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
if (empty($this->getInput('newspaper'))) {
|
||||
return static::URI . '/favicon.ico';
|
||||
}
|
||||
|
||||
return 'https://www.' . $this->getInput('newspaper') . '/favicon.ico';
|
||||
}
|
||||
|
||||
public function detectParameters($url)
|
||||
{
|
||||
$regex = '/^(https?:\/\/)?(www\.)?([a-z-]+\.fr)(\/)?([a-z-]+-[0-9]{5})?(\/)?$/';
|
||||
$url = strtolower($url);
|
||||
|
||||
if (preg_match($regex, $url, $urlMatches) === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!in_array($urlMatches[3], self::PARAMETERS['global']['newspaper']['values'], true)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return [
|
||||
'newspaper' => $urlMatches[3],
|
||||
'locality-slug' => empty($urlMatches[5]) ? null : $urlMatches[5]
|
||||
];
|
||||
}
|
||||
}
|
@ -18,25 +18,6 @@ class CeskaTelevizeBridge extends BridgeAbstract
|
||||
]
|
||||
];
|
||||
|
||||
private function fixChars($text)
|
||||
{
|
||||
return html_entity_decode($text, ENT_QUOTES, 'UTF-8');
|
||||
}
|
||||
|
||||
private function getUploadTimeFromString($string)
|
||||
{
|
||||
if (strpos($string, 'dnes') !== false) {
|
||||
return strtotime('today');
|
||||
} elseif (strpos($string, 'včera') !== false) {
|
||||
return strtotime('yesterday');
|
||||
} elseif (!preg_match('/(\d+).\s(\d+).(\s(\d+))?/', $string, $match)) {
|
||||
returnServerError('Could not get date from Česká televize string');
|
||||
}
|
||||
|
||||
$date = sprintf('%04d-%02d-%02d', $match[3] ?? date('Y'), $match[2], $match[1]);
|
||||
return strtotime($date);
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = $this->getInput('url');
|
||||
@ -58,24 +39,42 @@ class CeskaTelevizeBridge extends BridgeAbstract
|
||||
}
|
||||
|
||||
foreach ($html->find('#episodeListSection a[data-testid=card]') as $element) {
|
||||
$itemTitle = $element->find('h3', 0);
|
||||
$itemContent = $element->find('p[class^=content-]', 0);
|
||||
$itemDate = $element->find('div[class^=playTime-] span', 0);
|
||||
$itemThumbnail = $element->find('img', 0);
|
||||
$itemUri = self::URI . $element->getAttribute('href');
|
||||
$itemDate = $element->find('div[class^=playTime-] span, [data-testid=episode-item-broadcast] span', 0);
|
||||
|
||||
// Remove special characters and whitespace
|
||||
$cleanDate = preg_replace('/[^0-9.]/', '', $itemDate->plaintext);
|
||||
|
||||
$item = [
|
||||
'title' => $this->fixChars($itemTitle->plaintext),
|
||||
'uri' => $itemUri,
|
||||
'content' => '<img src="' . $itemThumbnail->getAttribute('src') . '" /><br />'
|
||||
. $this->fixChars($itemContent->plaintext),
|
||||
'timestamp' => $this->getUploadTimeFromString($itemDate->plaintext)
|
||||
'title' => $this->fixChars($element->find('h3', 0)->plaintext),
|
||||
'uri' => self::URI . $element->getAttribute('href'),
|
||||
'content' => '<img src="' . $element->find('img', 0)->getAttribute('srcset') . '" /><br />' . $this->fixChars($itemContent->plaintext),
|
||||
'timestamp' => $this->getUploadTimeFromString($cleanDate),
|
||||
];
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
private function getUploadTimeFromString($string)
|
||||
{
|
||||
if (strpos($string, 'dnes') !== false) {
|
||||
return strtotime('today');
|
||||
} elseif (strpos($string, 'včera') !== false) {
|
||||
return strtotime('yesterday');
|
||||
} elseif (!preg_match('/(\d+).(\d+).((\d+))?/', $string, $match)) {
|
||||
returnServerError('Could not get date from Česká televize string');
|
||||
}
|
||||
|
||||
$date = sprintf('%04d-%02d-%02d', $match[3] ?? date('Y'), $match[2], $match[1]);
|
||||
return strtotime($date);
|
||||
}
|
||||
|
||||
private function fixChars($text)
|
||||
{
|
||||
return html_entity_decode($text, ENT_QUOTES, 'UTF-8');
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
return $this->feedUri ?? parent::getURI();
|
||||
|
@ -2,59 +2,65 @@
|
||||
|
||||
class ComicsKingdomBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'stjohnjohnson';
|
||||
const MAINTAINER = 'TReKiE';
|
||||
// const MAINTAINER = 'stjohnjohnson';
|
||||
const NAME = 'Comics Kingdom Unofficial RSS';
|
||||
const URI = 'https://comicskingdom.com/';
|
||||
const URI = 'https://wp.comicskingdom.com/wp-json/wp/v2/ck_comic';
|
||||
const CACHE_TIMEOUT = 21600; // 6h
|
||||
const DESCRIPTION = 'Comics Kingdom Unofficial RSS';
|
||||
const PARAMETERS = [ [
|
||||
'comicname' => [
|
||||
'name' => 'comicname',
|
||||
'name' => 'Name of comic',
|
||||
'type' => 'text',
|
||||
'exampleValue' => 'mutts',
|
||||
'title' => 'The name of the comic in the URL after https://comicskingdom.com/',
|
||||
'required' => true
|
||||
],
|
||||
'limit' => [
|
||||
'name' => 'Limit',
|
||||
'type' => 'number',
|
||||
'title' => 'The number of recent comics to get',
|
||||
'defaultValue' => 10
|
||||
]
|
||||
]];
|
||||
|
||||
protected $comicName;
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM($this->getURI(), [], [], true, false);
|
||||
$json = getContents($this->getURI());
|
||||
$data = json_decode($json, false);
|
||||
|
||||
// Get author from first page
|
||||
$author = $html->find('div.author p', 0);
|
||||
;
|
||||
if (isset($data[0]->_embedded->{'wp:term'}[0][0])) {
|
||||
$this->comicName = $data[0]->_embedded->{'wp:term'}[0][0]->name;
|
||||
}
|
||||
|
||||
// Get current date/link
|
||||
$link = $html->find('meta[property=og:url]', -1)->content;
|
||||
for ($i = 0; $i < 3; $i++) {
|
||||
foreach ($data as $comicitem) {
|
||||
$item = [];
|
||||
|
||||
$page = getSimpleHTMLDOM($link);
|
||||
|
||||
$imagelink = $page->find('meta[property=og:image]', 0)->content;
|
||||
|
||||
$date = explode('/', $link);
|
||||
|
||||
$item['id'] = $imagelink;
|
||||
$item['uri'] = $link;
|
||||
$item['author'] = $author;
|
||||
$item['title'] = 'Comics Kingdom ' . $this->getInput('comicname');
|
||||
$item['timestamp'] = DateTime::createFromFormat('Y-m-d', $date[count($date) - 1])->getTimestamp();
|
||||
$item['content'] = '<img src="' . $imagelink . '" />';
|
||||
|
||||
$item['id'] = $comicitem->id;
|
||||
$item['uri'] = $comicitem->yoast_head_json->og_url;
|
||||
$item['author'] = str_ireplace('By ', '', $comicitem->ck_comic_byline);
|
||||
$item['title'] = $comicitem->yoast_head_json->title;
|
||||
$item['timestamp'] = $comicitem->date;
|
||||
$item['content'] = '<img src="' . $comicitem->yoast_head_json->og_image[0]->url . '" />';
|
||||
$this->items[] = $item;
|
||||
$link = $page->find('div.comic-viewer-inline a', 0)->href;
|
||||
if (empty($link)) {
|
||||
break; // allow bridge to continue if there's less than 3 comics
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
if (!is_null($this->getInput('comicname'))) {
|
||||
return self::URI . urlencode($this->getInput('comicname'));
|
||||
$params = [
|
||||
'ck_feature' => $this->getInput('comicname'),
|
||||
'per_page' => $this->getInput('limit'),
|
||||
'date_inclusive' => 'true',
|
||||
'order' => 'desc',
|
||||
'page' => '1',
|
||||
'_embed' => 'true'
|
||||
];
|
||||
|
||||
return self::URI . '?' . http_build_query($params);
|
||||
}
|
||||
|
||||
return parent::getURI();
|
||||
@ -62,8 +68,8 @@ class ComicsKingdomBridge extends BridgeAbstract
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (!is_null($this->getInput('comicname'))) {
|
||||
return $this->getInput('comicname') . ' - Comics Kingdom';
|
||||
if ($this->comicName) {
|
||||
return $this->comicName . ' - Comics Kingdom';
|
||||
}
|
||||
|
||||
return parent::getName();
|
||||
|
@ -109,7 +109,7 @@ class CrewbayBridge extends BridgeAbstract
|
||||
public function collectData()
|
||||
{
|
||||
$url = $this->getURI();
|
||||
$html = getSimpleHTMLDOM($url) or returnClientError('No results for this query.');
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
|
||||
$annonces = $html->find('#SearchResults div.result');
|
||||
$limit = 0;
|
||||
|
@ -442,7 +442,7 @@ class CssSelectorComplexBridge extends BridgeAbstract
|
||||
if (!is_null($time_selector) && $time_selector != '') {
|
||||
$time_element = $entry_html->find($time_selector, 0);
|
||||
$time = $time_element->getAttribute('datetime');
|
||||
if (is_null($time)) {
|
||||
if (empty($time)) {
|
||||
$time = $time_element->innertext;
|
||||
}
|
||||
|
||||
|
@ -47,8 +47,10 @@ class CubariBridge extends BridgeAbstract
|
||||
*/
|
||||
public function collectData()
|
||||
{
|
||||
// TODO: fix trivial SSRF
|
||||
$json = getContents($this->getInput('gist'));
|
||||
$jsonFile = json_decode($json, true);
|
||||
|
||||
$jsonFile = Json::decode($json);
|
||||
|
||||
$this->mangaTitle = $jsonFile['title'];
|
||||
|
||||
|
129
bridges/CubariProxyBridge.php
Normal file
129
bridges/CubariProxyBridge.php
Normal file
@ -0,0 +1,129 @@
|
||||
<?php
|
||||
|
||||
class CubariProxyBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Cubari Proxy';
|
||||
const MAINTAINER = 'phantop';
|
||||
const URI = 'https://cubari.moe';
|
||||
const DESCRIPTION = 'Returns chapters from Cubari.';
|
||||
const PARAMETERS = [[
|
||||
'service' => [
|
||||
'name' => 'Content service',
|
||||
'type' => 'list',
|
||||
'defaultValue' => 'mangadex',
|
||||
'values' => [
|
||||
'MangAventure' => 'mangadventure',
|
||||
'MangaDex' => 'mangadex',
|
||||
'MangaKatana' => 'mangakatana',
|
||||
'MangaSee' => 'mangasee',
|
||||
]
|
||||
],
|
||||
'series' => [
|
||||
'name' => 'Series ID/Name',
|
||||
'exampleValue' => '8c1d7d0c-e0b7-4170-941d-29f652c3c19d', # KnH
|
||||
'required' => true,
|
||||
],
|
||||
'fetch' => [
|
||||
'name' => 'Fetch chapter page images',
|
||||
'type' => 'list',
|
||||
'title' => 'Places chapter images in feed contents. Entries will consume more bandwidth.',
|
||||
'defaultValue' => 'c',
|
||||
'values' => [
|
||||
'None' => 'n',
|
||||
'Content' => 'c',
|
||||
'Enclosure' => 'e'
|
||||
]
|
||||
],
|
||||
'limit' => self::LIMIT
|
||||
]];
|
||||
|
||||
private $title;
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$limit = $this->getInput('limit') ?? 10;
|
||||
|
||||
$url = parent::getURI() . '/read/api/' . $this->getInput('service') . '/series/' . $this->getInput('series');
|
||||
$json = Json::decode(getContents($url));
|
||||
$this->title = $json['title'];
|
||||
|
||||
$chapters = $json['chapters'];
|
||||
krsort($chapters);
|
||||
|
||||
$count = 0;
|
||||
foreach ($chapters as $number => $element) {
|
||||
$item = [];
|
||||
$item['uri'] = $this->getURI() . '/' . $number;
|
||||
|
||||
if ($element['title']) {
|
||||
$item['title'] = $number . ' - ' . $element['title'];
|
||||
} else {
|
||||
$item['title'] = 'Volume ' . $element['volume'] . ' Chapter ' . $number;
|
||||
}
|
||||
|
||||
$group = '1';
|
||||
if (isset($element['release_date'])) {
|
||||
$dates = $element['release_date'];
|
||||
$date = max($dates);
|
||||
$item['timestamp'] = $date;
|
||||
$group = array_keys($dates, $date)[0];
|
||||
}
|
||||
$page = $element['groups'][$group];
|
||||
$item['author'] = $json['groups'][$group];
|
||||
$api = parent::getURI() . $page;
|
||||
$item['uid'] = $page;
|
||||
$item['comments'] = $api;
|
||||
|
||||
if ($this->getInput('fetch') != 'n') {
|
||||
$pages = [];
|
||||
try {
|
||||
$jsonp = getContents($api);
|
||||
$pages = Json::decode($jsonp);
|
||||
} catch (HttpException $e) {
|
||||
// allow error 500, as it's effectively a 429
|
||||
if ($e->getCode() != 500) {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
if ($this->getInput('fetch') == 'e') {
|
||||
$item['enclosures'] = $pages;
|
||||
}
|
||||
if ($this->getInput('fetch') == 'c') {
|
||||
$item['content'] = '';
|
||||
foreach ($pages as $img) {
|
||||
$item['content'] .= '<img src="' . $img . '"/>';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($count++ == $limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
$name = parent::getName();
|
||||
if (isset($this->title)) {
|
||||
$name .= ' - ' . $this->title;
|
||||
}
|
||||
return $name;
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
$uri = parent::getURI();
|
||||
if ($this->getInput('service')) {
|
||||
$uri .= '/read/' . $this->getInput('service') . '/' . $this->getInput('series');
|
||||
}
|
||||
return $uri;
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return parent::getURI() . '/static/favicon.png';
|
||||
}
|
||||
}
|
107
bridges/DRKBlutspendeBridge.php
Normal file
107
bridges/DRKBlutspendeBridge.php
Normal file
@ -0,0 +1,107 @@
|
||||
<?php
|
||||
|
||||
class DRKBlutspendeBridge extends FeedExpander
|
||||
{
|
||||
const MAINTAINER = 'User123698745';
|
||||
const NAME = 'DRK-Blutspende';
|
||||
const BASE_URI = 'https://www.drk-blutspende.de';
|
||||
const URI = self::BASE_URI;
|
||||
const CACHE_TIMEOUT = 60 * 60 * 1; // 1 hour
|
||||
const DESCRIPTION = 'German Red Cross (Deutsches Rotes Kreuz) blood donation service feed with more details';
|
||||
const CONTEXT_APPOINTMENTS = 'Termine';
|
||||
const PARAMETERS = [
|
||||
self::CONTEXT_APPOINTMENTS => [
|
||||
'term' => [
|
||||
'name' => 'PLZ / Ort',
|
||||
'required' => true,
|
||||
'exampleValue' => '12555',
|
||||
],
|
||||
'radius' => [
|
||||
'name' => 'Umkreis in km',
|
||||
'type' => 'number',
|
||||
'exampleValue' => 10,
|
||||
],
|
||||
'limit_days' => [
|
||||
'name' => 'Limit von Tagen',
|
||||
'title' => 'Nur Termine innerhalb der nächsten x Tagen',
|
||||
'type' => 'number',
|
||||
'exampleValue' => 28,
|
||||
],
|
||||
'limit_items' => [
|
||||
'name' => 'Limit von Terminen',
|
||||
'title' => 'Nicht mehr als x Termine',
|
||||
'type' => 'number',
|
||||
'required' => true,
|
||||
'defaultValue' => 20,
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$limitItems = intval($this->getInput('limit_items'));
|
||||
$this->collectExpandableDatas(self::buildAppointmentsURI(), $limitItems);
|
||||
}
|
||||
|
||||
protected function parseItem(array $item)
|
||||
{
|
||||
$html = getSimpleHTMLDOM($item['uri']);
|
||||
|
||||
$detailsElement = $html->find('.details', 0);
|
||||
|
||||
$dateElement = $detailsElement->find('.datum', 0);
|
||||
$dateLines = self::explodeLines($dateElement->plaintext);
|
||||
|
||||
$addressElement = $detailsElement->find('.adresse', 0);
|
||||
$addressLines = self::explodeLines($addressElement->plaintext);
|
||||
|
||||
$infoElement = $detailsElement->find('.angebote > h4 + p', 0);
|
||||
$info = $infoElement ? $infoElement->innertext : '';
|
||||
|
||||
$imageElements = $detailsElement->find('.fotos img');
|
||||
|
||||
$item['title'] = $dateLines[0] . ' ' . $dateLines[1] . ' ' . $addressLines[0] . ' - ' . $addressLines[1];
|
||||
|
||||
$item['content'] = <<<HTML
|
||||
<p><b>{$dateLines[0]} {$dateLines[1]}</b></p>
|
||||
<p>{$addressElement->innertext}</p>
|
||||
<p>{$info}</p>
|
||||
HTML;
|
||||
|
||||
foreach ($imageElements as $imageElement) {
|
||||
$src = $imageElement->getAttribute('src');
|
||||
$item['content'] .= <<<HTML
|
||||
<p><img src="{$src}"></p>
|
||||
HTML;
|
||||
}
|
||||
|
||||
$item['description'] = null;
|
||||
|
||||
return $item;
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
if ($this->queriedContext === self::CONTEXT_APPOINTMENTS) {
|
||||
return str_replace('.rss?', '?', self::buildAppointmentsURI());
|
||||
}
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
private function buildAppointmentsURI()
|
||||
{
|
||||
$term = $this->getInput('term') ?? '';
|
||||
$radius = $this->getInput('radius') ?? '';
|
||||
$limitDays = intval($this->getInput('limit_days'));
|
||||
$dateTo = $limitDays > 0 ? date('Y-m-d', time() + (60 * 60 * 24 * $limitDays)) : '';
|
||||
return self::BASE_URI . '/blutspendetermine/termine.rss?date_to=' . $dateTo . '&radius=' . $radius . '&term=' . $term;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an array of strings, each of which is a substring of string formed by splitting it on boundaries formed by line breaks.
|
||||
*/
|
||||
private function explodeLines(string $text): array
|
||||
{
|
||||
return array_map('trim', preg_split('/(\s*(\r\n|\n|\r)\s*)+/', $text));
|
||||
}
|
||||
}
|
102
bridges/DacksnackBridge.php
Normal file
102
bridges/DacksnackBridge.php
Normal file
@ -0,0 +1,102 @@
|
||||
<?PHP
|
||||
|
||||
class DacksnackBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Däcksnack';
|
||||
const URI = 'https://www.tidningendacksnack.se';
|
||||
const DESCRIPTION = 'Latest news by the magazine Däcksnack';
|
||||
const MAINTAINER = 'ajain-93';
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return self::URI . '/upload/favicon/2591047722.png';
|
||||
}
|
||||
|
||||
private function parseSwedishDates($dateString)
|
||||
{
|
||||
// Mapping of Swedish month names to English month names
|
||||
$monthNames = [
|
||||
'januari' => '01',
|
||||
'februari' => '02',
|
||||
'mars' => '03',
|
||||
'april' => '04',
|
||||
'maj' => '05',
|
||||
'juni' => '06',
|
||||
'juli' => '07',
|
||||
'augusti' => '08',
|
||||
'september' => '09',
|
||||
'oktober' => '10',
|
||||
'november' => '11',
|
||||
'december' => '12'
|
||||
];
|
||||
|
||||
// Split the date string into parts
|
||||
list($day, $monthName, $year) = explode(' ', $dateString);
|
||||
|
||||
// Convert month name to month number
|
||||
$month = $monthNames[$monthName];
|
||||
|
||||
// Format to a string recognizable by DateTime
|
||||
$formattedDate = sprintf('%04d-%02d-%02d', $year, $month, $day);
|
||||
|
||||
// Create a DateTime object
|
||||
$dateValue = new DateTime($formattedDate);
|
||||
|
||||
if ($dateValue) {
|
||||
$dateValue->setTime(0, 0); // Set time to 00:00
|
||||
return $dateValue->getTimestamp();
|
||||
}
|
||||
|
||||
return $dateValue ? $dateValue->getTimestamp() : false;
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$NEWSURL = self::URI;
|
||||
$html = getSimpleHTMLDOMCached($NEWSURL, 18000);
|
||||
|
||||
foreach ($html->find('a.main-news-item') as $element) {
|
||||
// Debug::log($element);
|
||||
|
||||
$title = trim($element->find('h2', 0)->plaintext);
|
||||
$category = trim($element->find('.category-tag', 0)->plaintext);
|
||||
$url = self::URI . $element->getAttribute('href');
|
||||
$published = $this->parseSwedishDates(trim($element->find('.published', 0)->plaintext));
|
||||
|
||||
$article_html = getSimpleHTMLDOMCached($url, 18000);
|
||||
$article_content = $article_html->find('#ctl00_ContentPlaceHolder1_NewsArticleVeiw_pnlArticle', 0);
|
||||
|
||||
$figure = self::URI . $article_content->find('img.news-image', 0)->getAttribute('src');
|
||||
$figure_caption = $article_content->find('.image-description', 0)->plaintext;
|
||||
$author = $article_content->find('span.main-article-author', 0)->plaintext;
|
||||
$preamble = $article_content->find('h4.main-article-ingress', 0)->plaintext;
|
||||
|
||||
$article_text = '';
|
||||
foreach ($article_content->find('div') as $div) {
|
||||
if (!$div->hasAttribute('class')) {
|
||||
$article_text = $div;
|
||||
}
|
||||
}
|
||||
|
||||
// Use a regular expression to extract the name
|
||||
if (preg_match('/Text:\s*(.*?)\s*Foto:/', $author, $matches)) {
|
||||
$author = $matches[1]; // This will contain 'Jonna Jansson'
|
||||
}
|
||||
|
||||
$content = '<b> [' . $category . '] <i>' . $preamble . '</i></b><br/><br/>';
|
||||
$content .= '<figure>';
|
||||
$content .= '<img src=' . $figure . '>';
|
||||
$content .= '<figcaption>' . $figure_caption . '</figcaption>';
|
||||
$content .= '</figure>';
|
||||
$content .= $article_text;
|
||||
|
||||
$this->items[] = [
|
||||
'uri' => $url,
|
||||
'title' => $title,
|
||||
'author' => $author,
|
||||
'timestamp' => $published,
|
||||
'content' => trim($content),
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
@ -18,8 +18,7 @@ class DagensNyheterDirektBridge extends BridgeAbstract
|
||||
{
|
||||
$NEWSURL = self::BASEURL . '/ajax/direkt/';
|
||||
|
||||
$html = getSimpleHTMLDOM($NEWSURL) or
|
||||
returnServerError('Could not request: ' . $NEWSURL);
|
||||
$html = getSimpleHTMLDOM($NEWSURL);
|
||||
|
||||
foreach ($html->find('article') as $element) {
|
||||
$link = $element->find('button', 0)->getAttribute('data-link');
|
||||
@ -27,11 +26,6 @@ class DagensNyheterDirektBridge extends BridgeAbstract
|
||||
$url = self::BASEURL . $link;
|
||||
$title = $element->find('h2', 0)->plaintext;
|
||||
$author = $element->find('div.ds-byline__titles', 0)->plaintext;
|
||||
// Debug::log($link);
|
||||
// Debug::log($datetime);
|
||||
// Debug::log($title);
|
||||
// Debug::log($url);
|
||||
// Debug::log($author);
|
||||
|
||||
$article_content = $element->find('div.direkt-post__content', 0);
|
||||
$article_html = '';
|
||||
|
96
bridges/DailythanthiBridge.php
Normal file
96
bridges/DailythanthiBridge.php
Normal file
@ -0,0 +1,96 @@
|
||||
<?php
|
||||
|
||||
class DailythanthiBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Dailythanthi';
|
||||
const URI = 'https://www.dailythanthi.com';
|
||||
const DESCRIPTION = 'Retrieve news from dailythanthi.com';
|
||||
const MAINTAINER = 'tillcash';
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'topic' => [
|
||||
'name' => 'topic',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'news' => [
|
||||
'tamilnadu' => 'news/state',
|
||||
'india' => 'news/india',
|
||||
'world' => 'news/world',
|
||||
'sirappu-katturaigal' => 'news/sirappukatturaigal',
|
||||
],
|
||||
'cinema' => [
|
||||
'news' => 'cinema/cinemanews',
|
||||
],
|
||||
'sports' => [
|
||||
'sports' => 'sports',
|
||||
'cricket' => 'sports/cricket',
|
||||
'football' => 'sports/football',
|
||||
'tennis' => 'sports/tennis',
|
||||
'hockey' => 'sports/hockey',
|
||||
'other-sports' => 'sports/othersports',
|
||||
],
|
||||
'devotional' => [
|
||||
'devotional' => 'others/devotional',
|
||||
'aalaya-varalaru' => 'aalaya-varalaru',
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
];
|
||||
|
||||
public function getName()
|
||||
{
|
||||
$topic = $this->getKey('topic');
|
||||
return self::NAME . ($topic ? ' - ' . ucfirst($topic) : '');
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$dom = getSimpleHTMLDOM(self::URI . '/' . $this->getInput('topic'));
|
||||
|
||||
foreach ($dom->find('div.ListingNewsWithMEDImage') as $element) {
|
||||
$slug = $element->find('a', 1);
|
||||
$title = $element->find('h3', 0);
|
||||
if (!$slug || !$title) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$url = self::URI . $slug->href;
|
||||
$date = $element->find('span', 1);
|
||||
$date = $date ? $date->{'data-datestring'} : '';
|
||||
|
||||
$this->items[] = [
|
||||
'content' => $this->constructContent($url),
|
||||
'timestamp' => $date ? $date . 'UTC' : '',
|
||||
'title' => $title->plaintext,
|
||||
'uid' => $slug->href,
|
||||
'uri' => $url,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
private function constructContent($url)
|
||||
{
|
||||
$dom = getSimpleHTMLDOMCached($url);
|
||||
|
||||
$article = $dom->find('div.details-content-story', 0);
|
||||
if (!$article) {
|
||||
return 'Content Not Found';
|
||||
}
|
||||
|
||||
// Remove ads
|
||||
foreach ($article->find('div[id*="_ad"]') as $remove) {
|
||||
$remove->outertext = '';
|
||||
}
|
||||
|
||||
// Correct image tag in $article
|
||||
foreach ($article->find('h-img') as $img) {
|
||||
$img->parent->outertext = sprintf('<p><img src="%s"></p>', $img->src);
|
||||
}
|
||||
|
||||
$image = $dom->find('div.main-image-caption-container img', 0);
|
||||
$image = $image ? '<p>' . $image->outertext . '</p>' : '';
|
||||
|
||||
return $image . $article;
|
||||
}
|
||||
}
|
@ -10,9 +10,11 @@ class DansTonChatBridge extends BridgeAbstract
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM(self::URI . 'latest.html');
|
||||
$url = self::URI . 'latest.html';
|
||||
$dom = getSimpleHTMLDOM($url);
|
||||
|
||||
foreach ($html->find('div.item') as $element) {
|
||||
$items = $dom->find('div.item');
|
||||
foreach ($items as $element) {
|
||||
$item = [];
|
||||
$item['uri'] = $element->find('a', 0)->href;
|
||||
$titleContent = $element->find('h3 a', 0);
|
||||
|
@ -9,7 +9,7 @@ class DarkReadingBridge extends FeedExpander
|
||||
|
||||
const PARAMETERS = [ [
|
||||
'feed' => [
|
||||
'name' => 'Feed',
|
||||
'name' => 'Feed (NOT IN USE)',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'All Dark Reading Stories' => '000_AllArticles',
|
||||
@ -41,17 +41,7 @@ class DarkReadingBridge extends FeedExpander
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$feed = $this->getInput('feed');
|
||||
$feed_splitted = explode('_', $feed);
|
||||
$feed_id = $feed_splitted[0];
|
||||
$feed_name = $feed_splitted[1];
|
||||
if (empty($feed) || !ctype_digit($feed_id) || !preg_match('/[A-Za-z%20\/]/', $feed_name)) {
|
||||
returnClientError('Invalid feed, please check the "feed" parameter.');
|
||||
}
|
||||
$feed_url = $this->getURI() . 'rss_simple.asp';
|
||||
if ($feed_id != '000') {
|
||||
$feed_url .= '?f_n=' . $feed_id . '&f_ln=' . $feed_name;
|
||||
}
|
||||
$feed_url = 'https://www.darkreading.com/rss.xml';
|
||||
$limit = $this->getInput('limit') ?? 10;
|
||||
$this->collectExpandableDatas($feed_url, $limit);
|
||||
}
|
||||
@ -71,7 +61,7 @@ class DarkReadingBridge extends FeedExpander
|
||||
|
||||
private function extractArticleContent($article)
|
||||
{
|
||||
$content = $article->find('div.article-content', 0)->innertext;
|
||||
$content = $article->find('div.ContentModule-Wrapper', 0)->innertext;
|
||||
|
||||
foreach (
|
||||
[
|
||||
|
@ -1,40 +0,0 @@
|
||||
<?php
|
||||
|
||||
class DavesTrailerPageBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'johnnygroovy';
|
||||
const NAME = 'Daves Trailer Page Bridge';
|
||||
const URI = 'https://www.davestrailerpage.co.uk/';
|
||||
const DESCRIPTION = 'Last trailers in HD thanks to Dave.';
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM(static::URI)
|
||||
or returnClientError('No results for this query.');
|
||||
|
||||
$curr_date = null;
|
||||
foreach ($html->find('tr') as $tr) {
|
||||
// If it's a date row, update the current date
|
||||
if ($tr->align == 'center') {
|
||||
$curr_date = $tr->plaintext;
|
||||
continue;
|
||||
}
|
||||
|
||||
$item = [];
|
||||
|
||||
// title
|
||||
$item['title'] = $tr->find('td', 0)->find('b', 0)->plaintext;
|
||||
|
||||
// content
|
||||
$item['content'] = $tr->find('ul', 1);
|
||||
|
||||
// uri
|
||||
$item['uri'] = $tr->find('a', 3)->getAttribute('href');
|
||||
|
||||
// date: parsed by FeedItem using strtotime
|
||||
$item['timestamp'] = $curr_date;
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
}
|
File diff suppressed because it is too large
Load Diff
@ -24,7 +24,8 @@ class DemosBerlinBridge extends BridgeAbstract
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$json = getContents('https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/index/all.json');
|
||||
$url = 'https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/index/all.json';
|
||||
$json = getContents($url);
|
||||
$jsonFile = json_decode($json, true);
|
||||
|
||||
$daysInterval = DateInterval::createFromDateString($this->getInput('days') . ' day');
|
||||
|
@ -78,13 +78,9 @@ class DerpibooruBridge extends BridgeAbstract
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$queryJson = json_decode(getContents(
|
||||
self::URI
|
||||
. 'api/v1/json/search/images?filter_id='
|
||||
. urlencode($this->getInput('f'))
|
||||
. '&q='
|
||||
. urlencode($this->getInput('q'))
|
||||
));
|
||||
$url = self::URI . 'api/v1/json/search/images?filter_id=' . urlencode($this->getInput('f')) . '&q=' . urlencode($this->getInput('q'));
|
||||
|
||||
$queryJson = json_decode(getContents($url));
|
||||
|
||||
foreach ($queryJson->images as $post) {
|
||||
$item = [];
|
||||
|
@ -73,12 +73,12 @@ class DeutscheWelleBridge extends FeedExpander
|
||||
|
||||
protected function parseItem(array $item)
|
||||
{
|
||||
$parsedUrl = parse_url($item['uri']);
|
||||
unset($parsedUrl['query']);
|
||||
$url = $this->unparseUrl($parsedUrl);
|
||||
$parsedUri = parse_url($item['uri']);
|
||||
unset($parsedUri['query']);
|
||||
$item['uri'] = $this->unparseUrl($parsedUri);
|
||||
|
||||
$page = getSimpleHTMLDOM($url);
|
||||
$page = defaultLinkTo($page, $url);
|
||||
$page = getSimpleHTMLDOM($item['uri']);
|
||||
$page = defaultLinkTo($page, $item['uri']);
|
||||
|
||||
$article = $page->find('article', 0);
|
||||
|
||||
@ -112,6 +112,13 @@ class DeutscheWelleBridge extends FeedExpander
|
||||
$img->height = null;
|
||||
}
|
||||
|
||||
// remove bad img src's added by defaultLinkTo() above
|
||||
// these images should have src="" and will then use
|
||||
// the srcset attribute to load the best image for the displayed size
|
||||
foreach ($article->find('figure > picture > img') as $img) {
|
||||
$img->src = '';
|
||||
}
|
||||
|
||||
// replace lazy-loaded images
|
||||
foreach ($article->find('figure.placeholder-image') as $figure) {
|
||||
$img = $figure->find('img', 0);
|
||||
|
@ -47,7 +47,7 @@ class DiarioDoAlentejoBridge extends BridgeAbstract
|
||||
}, self::PT_MONTH_NAMES),
|
||||
array_map(function ($num) {
|
||||
return sprintf('-%02d-', $num);
|
||||
}, range(1, sizeof(self::PT_MONTH_NAMES))),
|
||||
}, range(1, count(self::PT_MONTH_NAMES))),
|
||||
$element->find('span.date', 0)->innertext
|
||||
);
|
||||
|
||||
|
@ -1,5 +1,7 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
/**
|
||||
* Retourne les dons d'une recherche filtrée sur le site Donnons.org
|
||||
* Example: https://donnons.org/Sport/Ile-de-France
|
||||
@ -44,58 +46,60 @@ class DonnonsBridge extends BridgeAbstract
|
||||
{
|
||||
$uri = $this->getPageURI($page);
|
||||
|
||||
$html = getSimpleHTMLDOM($uri);
|
||||
$dom = getSimpleHTMLDOM($uri);
|
||||
|
||||
$searchDiv = $html->find('div[id=search]', 0);
|
||||
$searchDiv = $dom->find('div[id=search]', 0);
|
||||
|
||||
if (!is_null($searchDiv)) {
|
||||
$elements = $searchDiv->find('a.lst-annonce');
|
||||
foreach ($elements as $element) {
|
||||
$item = [];
|
||||
if (! $searchDiv) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Lien vers le don
|
||||
$item['uri'] = self::URI . $element->href;
|
||||
// Id de l'objet
|
||||
$item['uid'] = $element->getAttribute('data-id');
|
||||
$elements = $searchDiv->find('a.lst-annonce');
|
||||
foreach ($elements as $element) {
|
||||
$item = [];
|
||||
|
||||
// Grab info from json
|
||||
$jsonString = $element->find('script', 0)->innertext;
|
||||
$json = json_decode($jsonString, true);
|
||||
// Lien vers le don
|
||||
$item['uri'] = self::URI . $element->href;
|
||||
// Id de l'objet
|
||||
$item['uid'] = $element->getAttribute('data-id');
|
||||
|
||||
$name = $json['name'];
|
||||
$category = $json['category'];
|
||||
$date = $json['availabilityStarts'];
|
||||
$description = $json['description'];
|
||||
$city = $json['availableAtOrFrom']['address']['addressLocality'];
|
||||
$region = $json['availableAtOrFrom']['address']['addressRegion'];
|
||||
// Grab info from json
|
||||
$jsonString = $element->find('script', 0)->innertext;
|
||||
$json = json_decode($jsonString, true);
|
||||
|
||||
// Grab info from HTML
|
||||
$imageSrc = $element->find('img.ima-center', 0)->getAttribute('src');
|
||||
// Use large image instead of small one
|
||||
$imageSrc = str_replace('/xs/', '/lg/', $imageSrc);
|
||||
$image = self::URI . $imageSrc;
|
||||
$author = $element->find('div.avatar-holder', 0)->plaintext;
|
||||
$name = $json['name'];
|
||||
$category = $json['category'];
|
||||
$date = $json['availabilityStarts'];
|
||||
$description = $json['description'];
|
||||
$city = $json['availableAtOrFrom']['address']['addressLocality'];
|
||||
$region = $json['availableAtOrFrom']['address']['addressRegion'];
|
||||
|
||||
$content = '
|
||||
<img style="margin-right:1em;" src="' . $image . '">
|
||||
<div>
|
||||
<h1>' . $name . '</h1>
|
||||
<p>' . $description . '</p>
|
||||
<p>Lieu : <b>' . $city . '</b> - ' . $region . '</p>
|
||||
<p>Par : ' . $author . '</p>
|
||||
<p>Date : ' . $date . '</p>
|
||||
</div>
|
||||
';
|
||||
// Grab info from HTML
|
||||
$imageSrc = $element->find('img.ima-center', 0)->getAttribute('src');
|
||||
// Use large image instead of small one
|
||||
$imageSrc = str_replace('/xs/', '/lg/', $imageSrc);
|
||||
$image = self::URI . $imageSrc;
|
||||
$author = $element->find('div.avatar-holder', 0)->plaintext;
|
||||
|
||||
// Titre du don
|
||||
$item['title'] = '[' . $category . '] ' . $name;
|
||||
$item['timestamp'] = $date;
|
||||
$item['author'] = $author;
|
||||
$item['content'] = $content;
|
||||
$item['enclosures'] = [$image];
|
||||
$content = '
|
||||
<img style="margin-right:1em;" src="' . $image . '">
|
||||
<div>
|
||||
<h1>' . $name . '</h1>
|
||||
<p>' . $description . '</p>
|
||||
<p>Lieu : <b>' . $city . '</b> - ' . $region . '</p>
|
||||
<p>Par : ' . $author . '</p>
|
||||
<p>Date : ' . $date . '</p>
|
||||
</div>
|
||||
';
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
// Titre du don
|
||||
$item['title'] = '[' . $category . '] ' . $name;
|
||||
$item['timestamp'] = $date;
|
||||
$item['author'] = $author;
|
||||
$item['content'] = $content;
|
||||
$item['enclosures'] = [$image];
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -18,12 +18,12 @@ favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico';
|
||||
{
|
||||
$html = getSimpleHTMLDOM(self::URI);
|
||||
|
||||
$json = $this->loadEmbeddedJsonData($html);
|
||||
$data = $this->fetchData($html);
|
||||
|
||||
foreach ($html->find('li[id^="screenshot-"]') as $shot) {
|
||||
$item = [];
|
||||
|
||||
$additional_data = $this->findJsonForShot($shot, $json);
|
||||
$additional_data = $this->findJsonForShot($shot, $data);
|
||||
if ($additional_data === null) {
|
||||
$item['uri'] = self::URI . $shot->find('a', 0)->href;
|
||||
$item['title'] = $shot->find('.shot-title', 0)->plaintext;
|
||||
@ -46,9 +46,8 @@ favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico';
|
||||
}
|
||||
}
|
||||
|
||||
private function loadEmbeddedJsonData($html)
|
||||
private function fetchData($html)
|
||||
{
|
||||
$json = [];
|
||||
$scripts = $html->find('script');
|
||||
|
||||
foreach ($scripts as $script) {
|
||||
@ -69,12 +68,17 @@ favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico';
|
||||
$end = strpos($script->innertext, '];') + 1;
|
||||
|
||||
// convert JSON to PHP array
|
||||
$json = json_decode(substr($script->innertext, $start, $end - $start), true);
|
||||
break;
|
||||
$json = substr($script->innertext, $start, $end - $start);
|
||||
|
||||
try {
|
||||
// TODO: fix broken json
|
||||
return Json::decode($json);
|
||||
} catch (\JsonException $e) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $json;
|
||||
return [];
|
||||
}
|
||||
|
||||
private function findJsonForShot($shot, $json)
|
||||
|
86
bridges/DuvarOrgBridge.php
Normal file
86
bridges/DuvarOrgBridge.php
Normal file
@ -0,0 +1,86 @@
|
||||
<?php
|
||||
|
||||
class DuvarOrgBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Duvar.org - Haberler';
|
||||
const MAINTAINER = 'yourname';
|
||||
const URI = 'https://duvar.org';
|
||||
const DESCRIPTION = 'Returns the latest articles from Duvar.org - News from Turkey and the world';
|
||||
const CACHE_TIMEOUT = 3600; // 60min
|
||||
|
||||
const PARAMETERS = [[
|
||||
'postcount' => [
|
||||
'name' => 'Limit',
|
||||
'type' => 'number',
|
||||
'required' => true,
|
||||
'title' => 'Maximum number of items to return',
|
||||
'defaultValue' => 20,
|
||||
],
|
||||
'urlsuffix' => [
|
||||
'name' => 'URL Suffix',
|
||||
'type' => 'list',
|
||||
'title' => 'Suffix for the URL to scrape a specific section',
|
||||
'defaultValue' => 'Main',
|
||||
'values' => [
|
||||
'Main' => '',
|
||||
'Balanced' => '/uyumlu',
|
||||
'Protest' => '/muhalif',
|
||||
'Center' => '/merkez',
|
||||
'Alternative' => '/alternatif',
|
||||
'Global' => '/global',
|
||||
],
|
||||
],
|
||||
]];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$postCount = $this->getInput('postcount');
|
||||
$urlSuffix = $this->getInput('urlsuffix');
|
||||
$url = self::URI . $urlSuffix;
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
|
||||
foreach ($html->find('article.news-item') as $data) {
|
||||
if ($data === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
$item = [];
|
||||
$linkElement = $data->find('h2.news-title a', 0);
|
||||
$titleElement = $data->find('h2.news-title a', 0);
|
||||
$timestampElement = $data->find('time.meta-tag.date-tag', 0);
|
||||
$contentElement = $data->find('div.news-description', 0);
|
||||
|
||||
if ($linkElement) {
|
||||
$item['uri'] = $linkElement->getAttribute('href');
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
if ($titleElement) {
|
||||
$item['title'] = trim($titleElement->plaintext);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
if ($timestampElement) {
|
||||
$item['timestamp'] = strtotime($timestampElement->plaintext);
|
||||
} else {
|
||||
$item['timestamp'] = time();
|
||||
}
|
||||
if ($contentElement) {
|
||||
$item['content'] = trim($contentElement->plaintext);
|
||||
} else {
|
||||
$item['content'] = '';
|
||||
}
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
|
||||
$this->items[] = $item;
|
||||
|
||||
if (count($this->items) >= $postCount) {
|
||||
break;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
42
bridges/EASeedBridge.php
Normal file
42
bridges/EASeedBridge.php
Normal file
@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
class EASeedBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'EA Seed Blog';
|
||||
const URI = 'https://www.ea.com/seed';
|
||||
const DESCRIPTION = 'Posts from the EA Seed blog';
|
||||
const MAINTAINER = 'thefranke';
|
||||
const CACHE_TIMEOUT = 86400; // 24h
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$dom = getSimpleHTMLDOM(static::URI);
|
||||
$dom = $dom->find('ea-grid', 0);
|
||||
if (!$dom) {
|
||||
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
|
||||
}
|
||||
$dom = defaultLinkTo($dom, $this->getURI());
|
||||
foreach ($dom->find('ea-tile') as $article) {
|
||||
$a = $article->find('a', 0);
|
||||
$date = $article->find('div', 1)->plaintext;
|
||||
$title = $article->find('h3', 0)->plaintext;
|
||||
$author = $article->find('div', 0)->plaintext;
|
||||
|
||||
$entry = getSimpleHTMLDOMCached($a->href, static::CACHE_TIMEOUT * 7 * 4);
|
||||
|
||||
$content = $entry->find('main', 0);
|
||||
|
||||
// remove header and links to other posts
|
||||
$content->find('ea-header', 0)->outertext = '';
|
||||
$content->find('ea-section', -1)->outertext = '';
|
||||
|
||||
$this->items[] = [
|
||||
'title' => $title,
|
||||
'author' => $author,
|
||||
'uri' => $a->href,
|
||||
'content' => $content,
|
||||
'timestamp' => strtotime($date),
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
@ -5,15 +5,21 @@ class EBayBridge extends BridgeAbstract
|
||||
const NAME = 'eBay';
|
||||
const DESCRIPTION = 'Returns the search results from the eBay auctioning platforms';
|
||||
const URI = 'https://www.eBay.com';
|
||||
const MAINTAINER = 'wrobelda';
|
||||
const MAINTAINER = 'NotsoanoNimus, wrobelda';
|
||||
const PARAMETERS = [[
|
||||
'url' => [
|
||||
'name' => 'Search URL',
|
||||
'title' => 'Copy the URL from your browser\'s address bar after searching for your items and paste it here',
|
||||
'pattern' => '^(https:\/\/)?(www.)?ebay\.(com|com\.au|at|be|ca|ch|cn|es|fr|de|com\.hk|ie|it|com\.my|nl|ph|pl|com\.sg|co\.uk).*$',
|
||||
'pattern' => '^(https:\/\/)?(www\.)?(befr\.|benl\.)?ebay\.(com|com\.au|at|be|ca|ch|cn|es|fr|de|com\.hk|ie|it|com\.my|nl|ph|pl|com\.sg|co\.uk)\/.*$',
|
||||
'exampleValue' => 'https://www.ebay.com/sch/i.html?_nkw=atom+rss',
|
||||
'required' => true,
|
||||
]
|
||||
],
|
||||
'includesSearchLink' => [
|
||||
'name' => 'Include Original Search Link',
|
||||
'title' => 'Whether or not each feed item should include the original search query link to eBay which was used to find the given listing.',
|
||||
'type' => 'checkbox',
|
||||
'defaultValue' => false,
|
||||
],
|
||||
]];
|
||||
|
||||
public function getURI()
|
||||
@ -23,6 +29,10 @@ class EBayBridge extends BridgeAbstract
|
||||
$uri = trim(preg_replace('/([?&])_sop=[^&]+(&|$)/', '$1', $this->getInput('url')), '?&/');
|
||||
$uri .= (parse_url($uri, PHP_URL_QUERY) ? '&' : '?') . '_sop=10';
|
||||
|
||||
// Ensure the List View is used instead of the Gallery View.
|
||||
$uri = trim(preg_replace('/[?&]_dmd=[^&]+(&|$)/i', '$1', $uri), '?&/');
|
||||
$uri .= '&_dmd=1';
|
||||
|
||||
return $uri;
|
||||
} else {
|
||||
return parent::getURI();
|
||||
@ -46,7 +56,7 @@ class EBayBridge extends BridgeAbstract
|
||||
});
|
||||
|
||||
if ($searchQuery) {
|
||||
return $searchQuery[0];
|
||||
return 'eBay - ' . $searchQuery[0];
|
||||
}
|
||||
|
||||
return parent::getName();
|
||||
@ -61,44 +71,90 @@ class EBayBridge extends BridgeAbstract
|
||||
$inexactMatches->remove();
|
||||
}
|
||||
|
||||
// Remove "NEW LISTING" labels: we sort by the newest, so this is redundant.
|
||||
foreach ($html->find('.LIGHT_HIGHLIGHT') as $new_listing_label) {
|
||||
$new_listing_label->remove();
|
||||
}
|
||||
|
||||
$results = $html->find('ul.srp-results > li.s-item');
|
||||
foreach ($results as $listing) {
|
||||
$item = [];
|
||||
|
||||
// Remove "NEW LISTING" label, we sort by the newest, so this is redundant
|
||||
foreach ($listing->find('.LIGHT_HIGHLIGHT') as $new_listing_label) {
|
||||
$new_listing_label->remove();
|
||||
// Define a closure to shorten the ugliness of querying the current listing.
|
||||
$find = function ($query, $altText = '') use ($listing) {
|
||||
return $listing->find($query, 0)->plaintext ?? $altText;
|
||||
};
|
||||
|
||||
$item['title'] = $find('.s-item__title');
|
||||
if (!$item['title']) {
|
||||
// Skip entries where the title cannot be found (for w/e reason).
|
||||
continue;
|
||||
}
|
||||
|
||||
$listingTitle = $listing->find('.s-item__title', 0);
|
||||
if ($listingTitle) {
|
||||
$item['title'] = $listingTitle->plaintext;
|
||||
}
|
||||
|
||||
$subtitle = implode('', $listing->find('.s-item__subtitle'));
|
||||
|
||||
$listingUrl = $listing->find('.s-item__link', 0);
|
||||
if ($listingUrl) {
|
||||
$item['uri'] = $listingUrl->href;
|
||||
// It appears there may be more than a single 'subtitle' subclass in the listing. Collate them.
|
||||
$subtitles = $listing->find('.s-item__subtitle');
|
||||
if (is_array($subtitles)) {
|
||||
$subtitle = trim(implode(' ', array_column($subtitles, 'plaintext')));
|
||||
} else {
|
||||
$item['uri'] = null;
|
||||
$subtitle = trim($subtitles->plaintext ?? '');
|
||||
}
|
||||
|
||||
// Get the listing's link and uid.
|
||||
$itemUri = $listing->find('.s-item__link', 0);
|
||||
if ($itemUri) {
|
||||
$item['uri'] = $itemUri->href;
|
||||
}
|
||||
if (preg_match('/.*\/itm\/(\d+).*/i', $item['uri'], $matches)) {
|
||||
$item['uid'] = $matches[1];
|
||||
}
|
||||
|
||||
$priceDom = $listing->find('.s-item__details > .s-item__detail > .s-item__price', 0);
|
||||
$price = $priceDom->plaintext ?? 'N/A';
|
||||
// Price should be fetched on its own so we can provide the alt text without complication.
|
||||
$price = $find('.s-item__price', '[NO PRICE]');
|
||||
|
||||
$shippingFree = $listing->find('.s-item__details > .s-item__detail > .s-item__freeXDays', 0)->plaintext ?? '';
|
||||
$localDelivery = $listing->find('.s-item__details > .s-item__detail > .s-item__localDelivery', 0)->plaintext ?? '';
|
||||
$logisticsCost = $listing->find('.s-item__details > .s-item__detail > .s-item__logisticsCost', 0)->plaintext ?? '';
|
||||
// Map a list of dynamic variable names to their subclasses within the listing.
|
||||
// This is just a bit of sugar to make this cleaner and more maintainable.
|
||||
$propertyMappings = [
|
||||
'additionalPrice' => '.s-item__additional-price',
|
||||
'discount' => '.s-item__discount',
|
||||
'shippingFree' => '.s-item__freeXDays',
|
||||
'localDelivery' => '.s-item__localDelivery',
|
||||
'logisticsCost' => '.s-item__logisticsCost',
|
||||
'location' => '.s-item__location',
|
||||
'obo' => '.s-item__formatBestOfferEnabled',
|
||||
'sellerInfo' => '.s-item__seller-info-text',
|
||||
'bids' => '.s-item__bidCount',
|
||||
'timeLeft' => '.s-item__time-left',
|
||||
'timeEnd' => '.s-item__time-end',
|
||||
];
|
||||
|
||||
$location = $listing->find('.s-item__details > .s-item__detail > .s-item__location', 0)->plaintext ?? '';
|
||||
foreach ($propertyMappings as $k => $v) {
|
||||
$$k = $find($v);
|
||||
}
|
||||
|
||||
$sellerInfo = $listing->find('.s-item__seller-info-text', 0)->plaintext ?? '';
|
||||
// When an additional price detail or discount is defined, create the 'discountLine'.
|
||||
if ($additionalPrice || $discount) {
|
||||
$discountLine = '<br /><em>('
|
||||
. trim($additionalPrice ?? '')
|
||||
. '; ' . trim($discount ?? '')
|
||||
. ')</em>';
|
||||
} else {
|
||||
$discountLine = '';
|
||||
}
|
||||
|
||||
// Prepend the time-left info with a comma if the right details were found.
|
||||
$timeInfo = trim($timeLeft . ' ' . $timeEnd);
|
||||
if ($timeInfo) {
|
||||
$timeInfo = ', ' . $timeInfo;
|
||||
}
|
||||
|
||||
// Set the listing type.
|
||||
if ($bids) {
|
||||
$listingTypeDetails = "Auction: {$bids}{$timeInfo}";
|
||||
} else {
|
||||
$listingTypeDetails = 'Buy It Now';
|
||||
}
|
||||
|
||||
// Acquire the listing's primary image and atach it.
|
||||
$image = $listing->find('.s-item__image-wrapper > img', 0);
|
||||
if ($image) {
|
||||
// Not quite sure why append fragment here
|
||||
@ -106,11 +162,23 @@ class EBayBridge extends BridgeAbstract
|
||||
$item['enclosures'] = [$imageUrl];
|
||||
}
|
||||
|
||||
// Include the original search link, if specified.
|
||||
if ($this->getInput('includesSearchLink')) {
|
||||
$searchLink = '<p><small><a target="_blank" href="' . e($this->getURI()) . '">View Search</a></small></p>';
|
||||
} else {
|
||||
$searchLink = '';
|
||||
}
|
||||
|
||||
// Build the final item's content to display and add the item onto the list.
|
||||
$item['content'] = <<<CONTENT
|
||||
<p>$sellerInfo $location</p>
|
||||
<p><span style="font-weight:bold">$price</span> $shippingFree $localDelivery $logisticsCost<span></span></p>
|
||||
<p>$subtitle</p>
|
||||
<p><strong>$price</strong> $obo ($listingTypeDetails)
|
||||
$discountLine
|
||||
<br /><small>$shippingFree $localDelivery $logisticsCost</small></p>
|
||||
<p>{$subtitle}</p>
|
||||
$searchLink
|
||||
CONTENT;
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
@ -50,7 +50,9 @@ class EZTVBridge extends BridgeAbstract
|
||||
$eztv_uri = $this->getEztvUri();
|
||||
$ids = explode(',', trim($this->getInput('ids')));
|
||||
foreach ($ids as $id) {
|
||||
$data = json_decode(getContents(sprintf('%s/api/get-torrents?imdb_id=%s', $eztv_uri, $id)));
|
||||
$url = sprintf('%s/api/get-torrents?imdb_id=%s', $eztv_uri, $id);
|
||||
$json = getContents($url);
|
||||
$data = json_decode($json);
|
||||
if (!isset($data->torrents)) {
|
||||
// No results
|
||||
continue;
|
||||
|
@ -8,6 +8,12 @@ class EconomistBridge extends FeedExpander
|
||||
const CACHE_TIMEOUT = 3600; //1hour
|
||||
const DESCRIPTION = 'Returns the latest articles for the selected category';
|
||||
|
||||
const CONFIGURATION = [
|
||||
'cookie' => [
|
||||
'required' => false,
|
||||
]
|
||||
];
|
||||
|
||||
const PARAMETERS = [
|
||||
'global' => [
|
||||
'limit' => [
|
||||
@ -99,7 +105,24 @@ class EconomistBridge extends FeedExpander
|
||||
|
||||
protected function parseItem(array $item)
|
||||
{
|
||||
$dom = getSimpleHTMLDOM($item['uri']);
|
||||
$headers = [];
|
||||
if ($this->getOption('cookie')) {
|
||||
$headers = [
|
||||
'Authority: www.economist.com',
|
||||
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-language: en-US,en;q=0.9',
|
||||
'Cache-control: max-age=0',
|
||||
'Cookie: ' . $this->getOption('cookie'),
|
||||
'Upgrade-insecure-requests: 1',
|
||||
'User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36'
|
||||
];
|
||||
}
|
||||
try {
|
||||
$dom = getSimpleHTMLDOM($item['uri'], $headers);
|
||||
} catch (Exception $e) {
|
||||
$item['content'] = $e->getMessage();
|
||||
return $item;
|
||||
}
|
||||
|
||||
$article = $dom->find('#new-article-template', 0);
|
||||
if ($article == null) {
|
||||
@ -204,6 +227,15 @@ class EconomistBridge extends FeedExpander
|
||||
foreach ($elem->find('a.ds-link-with-arrow-icon') as $a) {
|
||||
$a->parent->removeChild($a);
|
||||
}
|
||||
// Sections like "Leaders on day X"
|
||||
foreach ($elem->find('div[data-tracking-id=content-well-chapter-list]') as $div) {
|
||||
$div->parent->removeChild($div);
|
||||
}
|
||||
// "Explore more" section
|
||||
foreach ($elem->find('h3[id=article-tags]') as $h3) {
|
||||
$div = $h3->parent;
|
||||
$div->parent->removeChild($div);
|
||||
}
|
||||
|
||||
// The Economist puts infographics into iframes, which doesn't
|
||||
// work in any of my readers. So this replaces iframes with
|
||||
|
@ -9,6 +9,12 @@ class EconomistWorldInBriefBridge extends BridgeAbstract
|
||||
const CACHE_TIMEOUT = 3600; // 1 hour
|
||||
const DESCRIPTION = 'Returns stories from the World in Brief section';
|
||||
|
||||
const CONFIGURATION = [
|
||||
'cookie' => [
|
||||
'required' => false,
|
||||
]
|
||||
];
|
||||
|
||||
const PARAMETERS = [
|
||||
'' => [
|
||||
'splitGobbets' => [
|
||||
@ -35,27 +41,51 @@ class EconomistWorldInBriefBridge extends BridgeAbstract
|
||||
'quote' => [
|
||||
'name' => 'Include the quote of the day',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'mergeEverything' => [
|
||||
'name' => 'Merge everything into one entry',
|
||||
'type' => 'checkbox',
|
||||
'defaultValue' => false,
|
||||
'title' => 'Whether to merge all the stories into one entry'
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM(self::URI);
|
||||
$gobbets = $html->find('._gobbets', 0);
|
||||
if ($this->getInput('splitGobbets') == 1) {
|
||||
$headers = [];
|
||||
if ($this->getOption('cookie')) {
|
||||
$headers = [
|
||||
'Authority: www.economist.com',
|
||||
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-language: en-US,en;q=0.9',
|
||||
'Cache-control: max-age=0',
|
||||
'Cookie: ' . $this->getOption('cookie'),
|
||||
'Upgrade-insecure-requests: 1',
|
||||
'User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36'
|
||||
];
|
||||
}
|
||||
$html = getSimpleHTMLDOM(self::URI, $headers);
|
||||
$gobbets = $html->find('p[data-component="the-world-in-brief-paragraph"]');
|
||||
if ($this->getInput('splitGobbets') == 1 && !$this->getInput('mergeEverything')) {
|
||||
$this->splitGobbets($gobbets);
|
||||
} else {
|
||||
$this->mergeGobbets($gobbets);
|
||||
};
|
||||
if ($this->getInput('agenda') == 1) {
|
||||
$articles = $html->find('._articles', 0);
|
||||
$this->collectArticles($articles);
|
||||
$articles = $html->find('div[data-test-id="chunks"] > div > div', 0);
|
||||
|
||||
if ($articles != null) {
|
||||
$this->collectArticles($articles);
|
||||
}
|
||||
}
|
||||
if ($this->getInput('quote') == 1) {
|
||||
$quote = $html->find('._quote-container', 0);
|
||||
$quote = $html->find('blockquote[data-test-id="inspirational-quote"]', 0);
|
||||
$this->addQuote($quote);
|
||||
}
|
||||
if ($this->getInput('mergeEverything') == 1) {
|
||||
$this->mergeEverything();
|
||||
}
|
||||
}
|
||||
|
||||
private function splitGobbets($gobbets)
|
||||
@ -63,7 +93,7 @@ class EconomistWorldInBriefBridge extends BridgeAbstract
|
||||
$today = new Datetime();
|
||||
$today->setTime(0, 0, 0, 0);
|
||||
$limit = $this->getInput('limit');
|
||||
foreach ($gobbets->find('._gobbet') as $gobbet) {
|
||||
foreach ($gobbets as $gobbet) {
|
||||
$title = $gobbet->plaintext;
|
||||
$match = preg_match('/[\.,]/', $title, $matches, PREG_OFFSET_CAPTURE);
|
||||
if ($match > 0) {
|
||||
@ -89,7 +119,7 @@ class EconomistWorldInBriefBridge extends BridgeAbstract
|
||||
$today = new Datetime();
|
||||
$today->setTime(0, 0, 0, 0);
|
||||
$contents = '';
|
||||
foreach ($gobbets->find('._gobbet') as $gobbet) {
|
||||
foreach ($gobbets as $gobbet) {
|
||||
$contents .= "<p>{$gobbet->innertext}";
|
||||
}
|
||||
$this->items[] = [
|
||||
@ -106,10 +136,17 @@ class EconomistWorldInBriefBridge extends BridgeAbstract
|
||||
$i = 0;
|
||||
$today = new Datetime();
|
||||
$today->setTime(0, 0, 0, 0);
|
||||
foreach ($articles->find('._article') as $article) {
|
||||
$title = $article->find('._headline', 0)->plaintext;
|
||||
$image = $article->find('._main-image', 0);
|
||||
$content = $article->find('._content', 0);
|
||||
foreach ($articles->children() as $element) {
|
||||
if ($element->tag != 'div') {
|
||||
continue;
|
||||
}
|
||||
if ($element->find('._newsletterContentPromo', 0) != null) {
|
||||
continue;
|
||||
}
|
||||
$image = $element->find('figure', 0);
|
||||
$title = $element->find('h3', 0)->plaintext;
|
||||
$content = $element->find('h3', 0)->parent();
|
||||
$content->find('h3', 0)->outertext = '';
|
||||
|
||||
$res_content = '';
|
||||
if ($image != null && $this->getInput('agendaPictures') == 1) {
|
||||
@ -140,4 +177,35 @@ class EconomistWorldInBriefBridge extends BridgeAbstract
|
||||
'uid' => 'quote-' . $today->format('U')
|
||||
];
|
||||
}
|
||||
|
||||
private function mergeEverything()
|
||||
{
|
||||
$today = new Datetime();
|
||||
$today->setTime(0, 0, 0, 0);
|
||||
$contents = '';
|
||||
|
||||
foreach ($this->items as $item) {
|
||||
$header = null;
|
||||
if (str_contains($item['uid'], 'story-')) {
|
||||
$header = $item['title'];
|
||||
} elseif (str_contains($item['uid'], 'quote-')) {
|
||||
$header = 'Quote of the day';
|
||||
} elseif (str_contains($item['uid'], 'world-in-brief-')) {
|
||||
$header = 'World in brief';
|
||||
}
|
||||
if ($header != null) {
|
||||
$contents .= "<h2>{$header}</h2>";
|
||||
}
|
||||
$contents .= $item['content'];
|
||||
}
|
||||
|
||||
$item = [
|
||||
'uri' => self::URI,
|
||||
'title' => 'The Economist World in Brief ' . $today->format('d.m.Y'),
|
||||
'content' => $contents,
|
||||
'timestamp' => $today->format('U'),
|
||||
'uid' => 'world-in-brief-merged' . $today->format('U')
|
||||
];
|
||||
$this->items = [$item];
|
||||
}
|
||||
}
|
||||
|
@ -12,8 +12,28 @@ class EdfPricesBridge extends BridgeAbstract
|
||||
'contract' => [
|
||||
'name' => 'Choisir un contrat',
|
||||
'type' => 'list',
|
||||
// we can add later HCHP, EJP, base
|
||||
'values' => ['Tempo' => '/energie/edf/tarifs/tempo'],
|
||||
// we can add later more option prices
|
||||
'values' => [
|
||||
'Base' => '/energie/edf/tarifs/tarif-bleu#base',
|
||||
'HPHC' => '/energie/edf/tarifs/tarif-bleu#hphc',
|
||||
'EJP' => '/energie/edf/tarifs/tarif-bleu#ejp',
|
||||
'Tempo' => '/energie/edf/tarifs/tempo'
|
||||
],
|
||||
],
|
||||
'power' => [
|
||||
'name' => 'Choisir une puissance',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'3 kVA' => 3,
|
||||
'6 kVA' => 6,
|
||||
'9 kVA' => 9,
|
||||
'12 kVA' => 12,
|
||||
'15 kVA' => 15,
|
||||
'18 kVA' => 18,
|
||||
'24 kVA' => 24,
|
||||
'30 kVA' => 30,
|
||||
'36 kVA' => 36
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
@ -24,36 +44,20 @@ class EdfPricesBridge extends BridgeAbstract
|
||||
* @param string $contractUri
|
||||
* @return void
|
||||
*/
|
||||
private function tempo(simple_html_dom $html, string $contractUri): void
|
||||
private function tempo(simple_html_dom $html, string $contractUri, int $power): void
|
||||
{
|
||||
// current color and next
|
||||
$daysDom = $html->find('#calendrier', 0)->nextSibling()->find('.card--ejp');
|
||||
if ($daysDom && count($daysDom) === 2) {
|
||||
foreach ($daysDom as $dayDom) {
|
||||
$day = trim($dayDom->find('.card__title', 0)->innertext) . '/' . (new \DateTime('now'))->format(('Y'));
|
||||
$dayColor = $dayDom->find('.card-ejp__icon span', 0)->innertext;
|
||||
|
||||
$text = $day . ' - ' . $dayColor;
|
||||
$item['uri'] = self::URI . $contractUri;
|
||||
$item['title'] = $text;
|
||||
$item['author'] = self::MAINTAINER;
|
||||
$item['content'] = $text;
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
// colors
|
||||
$ulDom = $html->find('#tarif-de-l-offre-edf-tempo-current-date-html-year', 0)->nextSibling()->nextSibling()->nextSibling();
|
||||
$ulDom = $html->find('#tarif-de-l-offre-tempo-edf-template-date-now-y', 0)->nextSibling()->nextSibling()->nextSibling();
|
||||
$elementsDom = $ulDom->find('li');
|
||||
if ($elementsDom && count($elementsDom) === 3) {
|
||||
// price per kWh is same for all powers
|
||||
foreach ($elementsDom as $elementDom) {
|
||||
$item = [];
|
||||
|
||||
$matches = [];
|
||||
preg_match_all('/Jour (.*) : Heures (.*) : (.*) € \/ Heures (.*) : (.*) €/um', $elementDom->innertext, $matches, PREG_SET_ORDER, 0);
|
||||
preg_match_all('/Jour (.*) : Heures (.*) : (.*) € \/ Heures (.*) : (.*) €/um', $elementDom->innertext, $matches, PREG_SET_ORDER, 0);
|
||||
|
||||
// for tempo contract we have 2x3 colors
|
||||
if ($matches && count($matches[0]) === 6) {
|
||||
for ($i = 0; $i < 2; $i++) {
|
||||
$text = 'Jour ' . $matches[0][1] . ' - Heures ' . $matches[0][2 + 2 * $i] . ' : ' . $matches[0][3 + 2 * $i] . '€';
|
||||
@ -69,26 +73,166 @@ class EdfPricesBridge extends BridgeAbstract
|
||||
}
|
||||
}
|
||||
|
||||
// powers
|
||||
$ulPowerContract = $ulDom->nextSibling()->nextSibling();
|
||||
$elementsPowerContractDom = $ulPowerContract->find('li');
|
||||
if ($elementsPowerContractDom && count($elementsPowerContractDom) === 4) {
|
||||
foreach ($elementsPowerContractDom as $elementPowerContractDom) {
|
||||
// add subscription power info
|
||||
$tablePrices = $ulDom->nextSibling()->nextSibling()->nextSibling()->find('.table--responsive', 0);
|
||||
$this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 7);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param simple_html_dom $html
|
||||
* @param string $contractUri
|
||||
* @return void
|
||||
*/
|
||||
private function base(simple_html_dom $html, string $contractUri, int $power): void
|
||||
{
|
||||
$tablePrices = $html
|
||||
->find('#grille-tarifaire-et-prix-du-kwh-du-tarif-reglemente-edf-en-option-base', 0)
|
||||
->nextSibling()
|
||||
->nextSibling()
|
||||
->nextSibling();
|
||||
|
||||
$prices = $tablePrices->find('.table--stripped tbody tr');
|
||||
// last element is useless because part of another table
|
||||
array_pop($prices);
|
||||
|
||||
// price per kWh is same for all powers
|
||||
if ($prices && count($prices) === 9) {
|
||||
$item = [];
|
||||
|
||||
$text = 'Base : ' . $prices[0]->children(2);
|
||||
$item['uri'] = self::URI . $contractUri;
|
||||
$item['title'] = $text;
|
||||
$item['author'] = self::MAINTAINER;
|
||||
$item['content'] = $text;
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
|
||||
$this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 9);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param simple_html_dom $html
|
||||
* @param string $contractUri
|
||||
* @return void
|
||||
*/
|
||||
private function hphc(simple_html_dom $html, string $contractUri, int $power): void
|
||||
{
|
||||
$tablePrices = $html
|
||||
->find('#grille-tarifaire-et-prix-du-kwh-du-tarif-reglemente-edf-en-option-heures-pleines-heures-creuses', 0)
|
||||
->nextSibling()
|
||||
->nextSibling()
|
||||
->nextSibling();
|
||||
|
||||
$prices = $tablePrices->find('.table--stripped tbody tr');
|
||||
// last element is useless because part of another table
|
||||
array_pop($prices);
|
||||
|
||||
// price per kWh is same for all powers
|
||||
if ($prices && count($prices) === 8) {
|
||||
$values = ['HC', 'HP'];
|
||||
foreach ($values as $key => $value) {
|
||||
$i++;
|
||||
$item = [];
|
||||
|
||||
$matches = [];
|
||||
preg_match_all('/(.*) kVA : (.*) €/um', $elementPowerContractDom->innertext, $matches, PREG_SET_ORDER, 0);
|
||||
$text = $values[$key] . ' : ' . $prices[0]->children($key + 2);
|
||||
$item['uri'] = self::URI . $contractUri;
|
||||
$item['title'] = $text;
|
||||
$item['author'] = self::MAINTAINER;
|
||||
$item['content'] = $text;
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
|
||||
if ($matches && count($matches[0]) === 3) {
|
||||
$text = $matches[0][1] . ' kVA : ' . $matches[0][2] . '€';
|
||||
$item['uri'] = self::URI . $contractUri;
|
||||
$item['title'] = $text;
|
||||
$item['author'] = self::MAINTAINER;
|
||||
$item['content'] = $text;
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
$this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param simple_html_dom $html
|
||||
* @param string $contractUri
|
||||
* @return void
|
||||
*/
|
||||
private function ejp(simple_html_dom $html, string $contractUri, int $power): void
|
||||
{
|
||||
$tablePrices = $html
|
||||
->find('#grille-tarifaire-et-prix-du-kwh-du-tarif-reglemente-edf-en-option-ejp', 0)
|
||||
->nextSibling()
|
||||
->nextSibling()
|
||||
->nextSibling();
|
||||
|
||||
$prices = $tablePrices->find('.table--stripped tbody tr');
|
||||
// last element is useless because part of another table
|
||||
array_pop($prices);
|
||||
|
||||
// price per kWh is same for all powers
|
||||
if ($prices && count($prices) === 5) {
|
||||
$values = ['Non EJP', 'EJP'];
|
||||
foreach ($values as $key => $value) {
|
||||
$i++;
|
||||
$item = [];
|
||||
|
||||
$text = $values[$key] . ' : ' . $prices[0]->children($key + 2);
|
||||
$item['uri'] = self::URI . $contractUri;
|
||||
$item['title'] = $text;
|
||||
$item['author'] = self::MAINTAINER;
|
||||
$item['content'] = $text;
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
$this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 5);
|
||||
}
|
||||
|
||||
private function addSubscriptionPowerInfo(simple_html_dom_node $tablePrices, string $contractUri, int $power, int $numberOfPrices): void
|
||||
{
|
||||
$prices = $tablePrices->find('.table--stripped tbody tr');
|
||||
// last element is useless because part of another table
|
||||
array_pop($prices);
|
||||
|
||||
// 7 contracts for tempo: 6, 9, 12, 15, 18, 30 and 36 kVA
|
||||
// 9 contracts for base: 3, 6, 9, 12, 15, 18, 24, 30 and 36 kVA
|
||||
// 7 contracts for HPHC: 6, 9, 12, 15, 18, 24, 30 and 36 kVA
|
||||
// 5 contracts for EJP: 9, 12, 15, 18 and 36 kVA
|
||||
if ($prices && count($prices) === $numberOfPrices) {
|
||||
$powerFound = false;
|
||||
foreach ($prices as $price) {
|
||||
$powerText = $price->firstChild()->firstChild()->innertext;
|
||||
$powerValue = (int)substr($powerText, 0, strpos($powerText, ' kVA'));
|
||||
|
||||
if ($powerValue !== $power) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$item = [];
|
||||
|
||||
$text = $powerText . ' : ' . $price->children(1) . '/an';
|
||||
$item['uri'] = self::URI . $contractUri;
|
||||
$item['title'] = $text;
|
||||
$item['author'] = self::MAINTAINER;
|
||||
$item['content'] = $text;
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
|
||||
$this->items[] = $item;
|
||||
$powerFound = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!$powerFound) {
|
||||
$item = [];
|
||||
|
||||
$text = 'Pas de tarif abonnement pour cette puissance et ce contrat';
|
||||
$item['uri'] = self::URI . $contractUri;
|
||||
$item['title'] = $text;
|
||||
$item['author'] = self::MAINTAINER;
|
||||
$item['content'] = $text;
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -97,10 +241,23 @@ class EdfPricesBridge extends BridgeAbstract
|
||||
{
|
||||
$contract = $this->getKey('contract');
|
||||
$contractUri = $this->getInput('contract');
|
||||
$power = $this->getInput('power');
|
||||
$html = getSimpleHTMLDOM(self::URI . $contractUri);
|
||||
|
||||
if ($contract === 'Tempo') {
|
||||
$this->tempo($html, $contractUri);
|
||||
$this->tempo($html, $contractUri, $power);
|
||||
}
|
||||
|
||||
if ($contract === 'Base') {
|
||||
$this->base($html, $contractUri, $power);
|
||||
}
|
||||
|
||||
if ($contract === 'HPHC') {
|
||||
$this->hphc($html, $contractUri, $power);
|
||||
}
|
||||
|
||||
if ($contract === 'EJP') {
|
||||
$this->ejp($html, $contractUri, $power);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -34,11 +34,9 @@ class ElloBridge extends BridgeAbstract
|
||||
];
|
||||
|
||||
if (!empty($this->getInput('u'))) {
|
||||
$postData = getContents(self::URI . 'api/v2/users/~' . urlencode($this->getInput('u')) . '/posts', $header) or
|
||||
returnServerError('Unable to query Ello API.');
|
||||
$postData = getContents(self::URI . 'api/v2/users/~' . urlencode($this->getInput('u')) . '/posts', $header);
|
||||
} else {
|
||||
$postData = getContents(self::URI . 'api/v2/posts?terms=' . urlencode($this->getInput('s')), $header) or
|
||||
returnServerError('Unable to query Ello API.');
|
||||
$postData = getContents(self::URI . 'api/v2/posts?terms=' . urlencode($this->getInput('s')), $header);
|
||||
}
|
||||
|
||||
$postData = json_decode($postData);
|
||||
@ -117,7 +115,7 @@ class ElloBridge extends BridgeAbstract
|
||||
$apiKey = $this->cache->get($cacheKey);
|
||||
|
||||
if (!$apiKey) {
|
||||
$keyInfo = getContents(self::URI . 'api/webapp-token') or returnServerError('Unable to get token.');
|
||||
$keyInfo = getContents(self::URI . 'api/webapp-token');
|
||||
$apiKey = json_decode($keyInfo)->token->access_token;
|
||||
$ttl = 60 * 60 * 20;
|
||||
$this->cache->set($cacheKey, $apiKey, $ttl);
|
||||
|
74
bridges/EpicGamesFreeBridge.php
Normal file
74
bridges/EpicGamesFreeBridge.php
Normal file
@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
class EpicGamesFreeBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Epic Games Free Games';
|
||||
const MAINTAINER = 'phantop';
|
||||
const URI = 'https://store.epicgames.com/';
|
||||
const DESCRIPTION = 'Returns the latest free games from Epic Games';
|
||||
const PARAMETERS = [ [
|
||||
'locale' => [
|
||||
'name' => 'Language',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'English' => 'en-US',
|
||||
'العربية' => 'ar',
|
||||
'Deutsch' => 'de',
|
||||
'Español (Spain)' => 'es-ES',
|
||||
'Español (LA)' => 'es-MX',
|
||||
'Français' => 'fr',
|
||||
'Italiano' => 'it',
|
||||
'日本語' => 'ja',
|
||||
'한국어' => 'ko',
|
||||
'Polski' => 'pl',
|
||||
'Português (Brasil)' => 'pt-BR',
|
||||
'Русский' => 'ru',
|
||||
'ไทย' => 'th',
|
||||
'Türkçe' => 'tr',
|
||||
'简体中文' => 'zh-CN',
|
||||
'繁體中文' => 'zh-Hant',
|
||||
],
|
||||
'title' => 'Language for game information',
|
||||
'defaultValue' => 'en-US',
|
||||
],
|
||||
'country' => [
|
||||
'name' => 'Country',
|
||||
'title' => 'Country store to check for deals',
|
||||
'defaultValue' => 'US',
|
||||
]
|
||||
]];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = 'https://store-site-backend-static.ak.epicgames.com/freeGamesPromotions?';
|
||||
$params = [
|
||||
'locale' => $this->getInput('locale'),
|
||||
'country' => $this->getInput('country'),
|
||||
'allowCountries' => $this->getInput('country'),
|
||||
];
|
||||
$url .= http_build_query($params);
|
||||
$json = Json::decode(getContents($url));
|
||||
|
||||
$data = $json['data']['Catalog']['searchStore']['elements'];
|
||||
foreach ($data as $element) {
|
||||
if (!isset($element['promotions']['promotionalOffers'][0])) {
|
||||
continue;
|
||||
}
|
||||
$item = [
|
||||
'author' => $element['seller']['name'],
|
||||
'content' => $element['description'],
|
||||
'enclosures' => array_map(fn($item) => $item['url'], $element['keyImages']),
|
||||
'timestamp' => strtotime($element['promotions']['promotionalOffers'][0]['promotionalOffers'][0]['startDate']),
|
||||
'title' => $element['title'],
|
||||
'url' => parent::getURI() . $this->getInput('locale') . '/p/' . $element['urlSlug'],
|
||||
];
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
$uri = parent::getURI() . $this->getInput('locale') . '/free-games';
|
||||
return $uri;
|
||||
}
|
||||
}
|
@ -31,7 +31,7 @@ class FDroidBridge extends BridgeAbstract
|
||||
CURLOPT_NOBODY => true,
|
||||
];
|
||||
$reponse = getContents($url, [], $curlOptions, true);
|
||||
$lastModified = $reponse['headers']['last-modified'][0] ?? null;
|
||||
$lastModified = $reponse->getHeader('last-modified');
|
||||
$timestamp = strtotime($lastModified ?? 'today');
|
||||
return $timestamp;
|
||||
}
|
||||
|
@ -14,7 +14,7 @@ class FDroidRepoBridge extends BridgeAbstract
|
||||
'name' => 'Repository URL',
|
||||
'title' => 'Usually ends with /repo/',
|
||||
'required' => true,
|
||||
'exampleValue' => 'https://srv.tt-rss.org/fdroid/repo'
|
||||
'exampleValue' => 'https://molly.im/fdroid/foss/fdroid/repo'
|
||||
]
|
||||
],
|
||||
'Latest Updates' => [
|
||||
@ -35,7 +35,7 @@ class FDroidRepoBridge extends BridgeAbstract
|
||||
'package' => [
|
||||
'name' => 'Package Identifier',
|
||||
'required' => true,
|
||||
'exampleValue' => 'org.fox.ttrss'
|
||||
'exampleValue' => 'im.molly.app'
|
||||
]
|
||||
]
|
||||
];
|
||||
@ -45,11 +45,7 @@ class FDroidRepoBridge extends BridgeAbstract
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
if (!extension_loaded('zip')) {
|
||||
throw new \Exception('FDroidRepoBridge requires the php-zip extension');
|
||||
}
|
||||
|
||||
$this->repo = $this->getRepo();
|
||||
$this->repo = $this->fetchData();
|
||||
switch ($this->queriedContext) {
|
||||
case 'Latest Updates':
|
||||
$this->getAllUpdates();
|
||||
@ -58,63 +54,15 @@ class FDroidRepoBridge extends BridgeAbstract
|
||||
$this->getPackage($this->getInput('package'));
|
||||
break;
|
||||
default:
|
||||
returnServerError('Unimplemented Context (collectData)');
|
||||
throw new \Exception('Unimplemented Context (collectData)');
|
||||
}
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
if (empty($this->queriedContext)) {
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
$url = rtrim($this->GetInput('url'), '/');
|
||||
return strstr($url, '?', true) ?: $url;
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (empty($this->queriedContext)) {
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
$name = $this->repo['repo']['name'];
|
||||
switch ($this->queriedContext) {
|
||||
case 'Latest Updates':
|
||||
return $name;
|
||||
case 'Follow Package':
|
||||
return $this->getInput('package') . ' - ' . $name;
|
||||
default:
|
||||
returnServerError('Unimplemented Context (getName)');
|
||||
}
|
||||
}
|
||||
|
||||
private function getRepo()
|
||||
private function fetchData()
|
||||
{
|
||||
$url = $this->getURI();
|
||||
|
||||
// Get repo information (only available as JAR)
|
||||
$jar = getContents($url . '/index-v1.jar');
|
||||
$jar_loc = tempnam(sys_get_temp_dir(), '');
|
||||
file_put_contents($jar_loc, $jar);
|
||||
|
||||
// JAR files are specially formatted ZIP files
|
||||
$jar = new \ZipArchive();
|
||||
if ($jar->open($jar_loc) !== true) {
|
||||
unlink($jar_loc);
|
||||
throw new \Exception('Failed to extract archive');
|
||||
}
|
||||
|
||||
// Get file pointer to the relevant JSON inside
|
||||
$fp = $jar->getStream('index-v1.json');
|
||||
if (!$fp) {
|
||||
returnServerError('Failed to get file pointer');
|
||||
}
|
||||
|
||||
$data = json_decode(stream_get_contents($fp), true);
|
||||
fclose($fp);
|
||||
$jar->close();
|
||||
unlink($jar_loc);
|
||||
$json = getContents($url . '/index-v1.json');
|
||||
$data = Json::decode($json);
|
||||
return $data;
|
||||
}
|
||||
|
||||
@ -158,9 +106,9 @@ class FDroidRepoBridge extends BridgeAbstract
|
||||
$summary = $lang['summary'] ?? $app['summary'] ?? '';
|
||||
$description = markdownToHtml(trim($lang['description'] ?? $app['description'] ?? 'None'));
|
||||
$whatsNew = markdownToHtml(trim($lang['whatsNew'] ?? 'None'));
|
||||
$website = $this->link($lang['webSite'] ?? $app['webSite'] ?? $app['authorWebSite'] ?? null);
|
||||
$source = $this->link($app['sourceCode'] ?? null);
|
||||
$issueTracker = $this->link($app['issueTracker'] ?? null);
|
||||
$website = $this->createAnchor($lang['webSite'] ?? $app['webSite'] ?? $app['authorWebSite'] ?? null);
|
||||
$source = $this->createAnchor($app['sourceCode'] ?? null);
|
||||
$issueTracker = $this->createAnchor($app['issueTracker'] ?? null);
|
||||
$license = $app['license'] ?? 'None';
|
||||
$item['content'] = <<<EOD
|
||||
{$icon}
|
||||
@ -182,7 +130,7 @@ EOD;
|
||||
private function getPackage($package)
|
||||
{
|
||||
if (!isset($this->repo['packages'][$package])) {
|
||||
returnClientError('Invalid Package Name');
|
||||
throw new \Exception('Invalid Package Name');
|
||||
}
|
||||
$package = $this->repo['packages'][$package];
|
||||
|
||||
@ -192,7 +140,7 @@ EOD;
|
||||
$item['uri'] = $this->getURI() . '/' . $version['apkName'];
|
||||
$item['title'] = $version['versionName'];
|
||||
$item['timestamp'] = date(DateTime::ISO8601, (int) ($version['added'] / 1000));
|
||||
$item['uid'] = $version['versionCode'];
|
||||
$item['uid'] = (string) $version['versionCode'];
|
||||
$size = round($version['size'] / 1048576, 1); // Bytes -> MB
|
||||
$sdk_link = 'https://developer.android.com/studio/releases/platforms';
|
||||
$item['content'] = <<<EOD
|
||||
@ -208,11 +156,42 @@ EOD;
|
||||
}
|
||||
}
|
||||
|
||||
private function link($url)
|
||||
public function getURI()
|
||||
{
|
||||
if (empty($this->queriedContext)) {
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
$url = rtrim($this->getInput('url'), '/');
|
||||
if (strstr($url, '?', true)) {
|
||||
return strstr($url, '?', true);
|
||||
} else {
|
||||
return $url;
|
||||
}
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (empty($this->queriedContext)) {
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
$name = $this->repo['repo']['name'];
|
||||
switch ($this->queriedContext) {
|
||||
case 'Latest Updates':
|
||||
return $name;
|
||||
case 'Follow Package':
|
||||
return $this->getInput('package') . ' - ' . $name;
|
||||
default:
|
||||
throw new \Exception('Unimplemented Context (getName)');
|
||||
}
|
||||
}
|
||||
|
||||
private function createAnchor($url)
|
||||
{
|
||||
if (empty($url)) {
|
||||
return null;
|
||||
}
|
||||
return '<a href="' . $url . '">' . $url . '</a>';
|
||||
return sprintf('<a href="%s">%s</a>', $url, $url);
|
||||
}
|
||||
}
|
||||
|
@ -6,8 +6,10 @@ class FeedMergeBridge extends FeedExpander
|
||||
const NAME = 'FeedMerge';
|
||||
const URI = 'https://github.com/RSS-Bridge/rss-bridge';
|
||||
const DESCRIPTION = <<<'TEXT'
|
||||
This bridge merges two or more feeds into a single feed. Max 10 items are fetched from each feed.
|
||||
TEXT;
|
||||
This bridge merges two or more feeds into a single feed. <br>
|
||||
Max 10 latest items are fetched from each individual feed. <br>
|
||||
Items with identical url or title are considered duplicates (and are removed). <br>
|
||||
TEXT;
|
||||
|
||||
const PARAMETERS = [
|
||||
[
|
||||
@ -36,11 +38,11 @@ TEXT;
|
||||
];
|
||||
|
||||
/**
|
||||
* todo: Consider a strategy which produces a shorter feed url
|
||||
* TODO: Consider a strategy which produces a shorter feed url
|
||||
*/
|
||||
public function collectData()
|
||||
{
|
||||
$limit = (int)($this->getInput('limit') ?: 10);
|
||||
$limit = (int)($this->getInput('limit') ?: 99);
|
||||
$feeds = [
|
||||
$this->getInput('feed_1'),
|
||||
$this->getInput('feed_2'),
|
||||
@ -61,9 +63,10 @@ TEXT;
|
||||
if (count($feeds) > 1) {
|
||||
// Allow one or more feeds to fail
|
||||
try {
|
||||
$this->collectExpandableDatas($feed);
|
||||
$this->collectExpandableDatas($feed, 10);
|
||||
} catch (HttpException $e) {
|
||||
$this->logger->warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e)));
|
||||
// This feed item might be spammy. Considering dropping it.
|
||||
$this->items[] = [
|
||||
'title' => 'RSS-Bridge: ' . $e->getMessage(),
|
||||
// Give current time so it sorts to the top
|
||||
@ -71,7 +74,7 @@ TEXT;
|
||||
];
|
||||
continue;
|
||||
} catch (\Exception $e) {
|
||||
if (str_starts_with($e->getMessage(), 'Unable to parse xml')) {
|
||||
if (str_starts_with($e->getMessage(), 'Failed to parse xml')) {
|
||||
// Allow this particular exception from FeedExpander
|
||||
$this->logger->warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e)));
|
||||
continue;
|
||||
@ -79,29 +82,48 @@ TEXT;
|
||||
throw $e;
|
||||
}
|
||||
} else {
|
||||
$this->collectExpandableDatas($feed);
|
||||
$this->collectExpandableDatas($feed, 10);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by timestamp descending
|
||||
// If $this->items is empty we should consider throw exception here
|
||||
|
||||
// Sort by timestamp, uri, title in descending order
|
||||
usort($this->items, function ($a, $b) {
|
||||
$t1 = $a['timestamp'] ?? $a['uri'] ?? $a['title'];
|
||||
$t2 = $b['timestamp'] ?? $b['uri'] ?? $b['title'];
|
||||
return $t2 <=> $t1;
|
||||
});
|
||||
|
||||
// Remove duplicates by using url as unique key
|
||||
// Remove duplicates by url
|
||||
$items = [];
|
||||
foreach ($this->items as $item) {
|
||||
$index = $item['uri'] ?? null;
|
||||
if ($index) {
|
||||
// Overwrite duplicates
|
||||
$items[$index] = $item;
|
||||
$uri = $item['uri'] ?? null;
|
||||
if ($uri) {
|
||||
// Insert or override the existing duplicate
|
||||
$items[$uri] = $item;
|
||||
} else {
|
||||
// The item doesn't have a uri!
|
||||
$items[] = $item;
|
||||
}
|
||||
}
|
||||
$this->items = array_slice(array_values($items), 0, $limit);
|
||||
$this->items = array_values($items);
|
||||
|
||||
// Remove duplicates by title
|
||||
$items = [];
|
||||
foreach ($this->items as $item) {
|
||||
$title = $item['title'] ?? null;
|
||||
if ($title) {
|
||||
// Insert or override the existing duplicate
|
||||
$items[$title] = $item;
|
||||
} else {
|
||||
// The item doesn't have a title!
|
||||
$items[] = $item;
|
||||
}
|
||||
}
|
||||
$this->items = array_values($items);
|
||||
|
||||
$this->items = array_slice($this->items, 0, $limit);
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
|
@ -187,7 +187,6 @@ class FicbookBridge extends BridgeAbstract
|
||||
$fixed_date = str_replace(' г.', '', $fixed_date);
|
||||
|
||||
if ($fixed_date === $date) {
|
||||
Debug::log('Unable to fix date: ' . $date);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -15,6 +15,12 @@ class FilterBridge extends FeedExpander
|
||||
'exampleValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day',
|
||||
'required' => true,
|
||||
],
|
||||
'name' => [
|
||||
'name' => 'Feed name (optional)',
|
||||
'type' => 'text',
|
||||
'exampleValue' => 'My feed',
|
||||
'required' => false,
|
||||
],
|
||||
'filter' => [
|
||||
'name' => 'Filter (regular expression!!!)',
|
||||
'required' => false,
|
||||
@ -77,7 +83,7 @@ class FilterBridge extends FeedExpander
|
||||
{
|
||||
$url = $this->getInput('url');
|
||||
if (!Url::validate($url)) {
|
||||
returnClientError('The url parameter must either refer to http or https protocol.');
|
||||
throw new \Exception('The url parameter must either refer to http or https protocol.');
|
||||
}
|
||||
$this->collectExpandableDatas($this->getURI());
|
||||
}
|
||||
@ -158,11 +164,18 @@ class FilterBridge extends FeedExpander
|
||||
public function getURI()
|
||||
{
|
||||
$url = $this->getInput('url');
|
||||
|
||||
if (empty($url)) {
|
||||
$url = parent::getURI();
|
||||
if ($url) {
|
||||
return $url;
|
||||
}
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
return $url;
|
||||
public function getName()
|
||||
{
|
||||
$name = $this->getInput('name');
|
||||
if ($name) {
|
||||
return $name;
|
||||
}
|
||||
return parent::getName();
|
||||
}
|
||||
}
|
||||
|
@ -60,7 +60,7 @@ class FindACrewBridge extends BridgeAbstract
|
||||
CURLOPT_POSTFIELDS => http_build_query($data) . "\n"
|
||||
];
|
||||
|
||||
$html = getSimpleHTMLDOM($url, $header, $opts) or returnClientError('No results for this query.');
|
||||
$html = getSimpleHTMLDOM($url, $header, $opts);
|
||||
|
||||
$annonces = $html->find('.css_SrhRst');
|
||||
$limit = $this->getInput('limit') ?? 10;
|
||||
|
@ -44,8 +44,6 @@ class FolhaDeSaoPauloBridge extends FeedExpander
|
||||
$item['content'] = $text;
|
||||
$item['uri'] = explode('*', $item['uri'])[1];
|
||||
}
|
||||
} else {
|
||||
Debug::log('???: ' . $item['uri']);
|
||||
}
|
||||
} else {
|
||||
$item['uri'] = explode('*', $item['uri'])[1];
|
||||
@ -58,13 +56,11 @@ class FolhaDeSaoPauloBridge extends FeedExpander
|
||||
{
|
||||
$feed_input = $this->getInput('feed');
|
||||
if (substr($feed_input, 0, strlen(self::URI)) === self::URI) {
|
||||
Debug::log('Input:: ' . $feed_input);
|
||||
$feed_url = $feed_input;
|
||||
} else {
|
||||
/* TODO: prepend `/` if missing */
|
||||
$feed_url = self::URI . '/' . $this->getInput('feed');
|
||||
}
|
||||
Debug::log('URL: ' . $feed_url);
|
||||
$limit = $this->getInput('amount');
|
||||
$this->collectExpandableDatas($feed_url, $limit);
|
||||
}
|
||||
|
25
bridges/ForensicArchitectureBridge.php
Normal file
25
bridges/ForensicArchitectureBridge.php
Normal file
@ -0,0 +1,25 @@
|
||||
<?php
|
||||
|
||||
class ForensicArchitectureBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Forensic Architecture';
|
||||
const URI = 'https://forensic-architecture.org/';
|
||||
const DESCRIPTION = 'Generates content feeds from forensic-architecture.org';
|
||||
const MAINTAINER = 'tillcash';
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = 'https://forensic-architecture.org/api/fa/v1/investigations';
|
||||
$jsonData = json_decode(getContents($url));
|
||||
|
||||
foreach ($jsonData->investigations as $investigation) {
|
||||
$this->items[] = [
|
||||
'content' => $investigation->abstract,
|
||||
'timestamp' => $investigation->publication_date,
|
||||
'title' => $investigation->title,
|
||||
'uid' => $investigation->id,
|
||||
'uri' => self::URI . 'investigation/' . $investigation->slug,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
@ -5,13 +5,13 @@ class Formula1Bridge extends BridgeAbstract
|
||||
const NAME = 'Formula1 Bridge';
|
||||
const URI = 'https://formula1.com/';
|
||||
const DESCRIPTION = 'Returns latest official Formula 1 news';
|
||||
const MAINTAINER = 'AxorPL';
|
||||
const MAINTAINER = 'axor-mst';
|
||||
|
||||
const API_KEY = 'qPgPPRJyGCIPxFT3el4MF7thXHyJCzAP';
|
||||
const API_KEY = 'xZ7AOODSjiQadLsIYWefQrpCSQVDbHGC';
|
||||
const API_URL = 'https://api.formula1.com/v1/editorial/articles?limit=%u';
|
||||
|
||||
const ARTICLE_AUTHOR = 'Formula 1';
|
||||
const ARTICLE_URL = 'https://formula1.com/en/latest/article.%s.%s.html';
|
||||
const ARTICLE_URL = 'https://formula1.com/en/latest/article/%s.%s';
|
||||
|
||||
const LIMIT_MIN = 1;
|
||||
const LIMIT_DEFAULT = 10;
|
||||
@ -36,7 +36,11 @@ class Formula1Bridge extends BridgeAbstract
|
||||
$limit = min(self::LIMIT_MAX, max(self::LIMIT_MIN, $limit));
|
||||
$url = sprintf(self::API_URL, $limit);
|
||||
|
||||
$json = json_decode(getContents($url, ['apikey: ' . self::API_KEY]));
|
||||
$json = json_decode(getContents($url, [
|
||||
'Accept: application/json',
|
||||
'apikey: ' . self::API_KEY,
|
||||
'locale: en'
|
||||
]));
|
||||
if (property_exists($json, 'error')) {
|
||||
returnServerError($json->message);
|
||||
}
|
||||
|
@ -3,7 +3,8 @@
|
||||
class FreeTelechargerBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Free-Telecharger';
|
||||
const URI = 'https://www.free-telecharger.art/';
|
||||
const URI = 'https://www.free-telecharger.fun/';
|
||||
const ALTERNATEURI = 'https://www.free-telecharger.com/';
|
||||
const DESCRIPTION = 'Suivi de série sur Free-Telecharger';
|
||||
const MAINTAINER = 'sysadminstory';
|
||||
const PARAMETERS = [
|
||||
@ -12,19 +13,19 @@ class FreeTelechargerBridge extends BridgeAbstract
|
||||
'name' => 'URL de la série',
|
||||
'type' => 'text',
|
||||
'required' => true,
|
||||
'title' => 'URL d\'une série sans le https://www.free-telecharger.art/',
|
||||
'title' => 'URL d\'une série sans le https://www.free-telecharger.fun/',
|
||||
'pattern' => 'series.*\.html',
|
||||
'exampleValue' => 'series-vf-hd/151432-wolf-saison-1-complete-web-dl-720p.html'
|
||||
],
|
||||
]
|
||||
];
|
||||
const CACHE_TIMEOUT = 3600;
|
||||
private string $showTitle;
|
||||
private string $showTechDetails;
|
||||
private string $showTitle = '';
|
||||
private string $showTechDetails = '';
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM(self::URI . $this->getInput('url'));
|
||||
$html = getSimpleHTMLDOM(self::ALTERNATEURI . $this->getInput('url'));
|
||||
|
||||
// Find all block content of the page
|
||||
$blocks = $html->find('div[class=block1]');
|
||||
|
@ -32,7 +32,7 @@ class FunkBridge extends BridgeAbstract
|
||||
$url .= '?size=' . $this->getInput('max');
|
||||
}
|
||||
|
||||
$jsonString = getContents($url) or returnServerError('No contents received!');
|
||||
$jsonString = getContents($url);
|
||||
$json = json_decode($jsonString, true);
|
||||
|
||||
foreach ($json['list'] as $element) {
|
||||
|
@ -676,7 +676,7 @@ class FurAffinityBridge extends BridgeAbstract
|
||||
$name = parent::getName();
|
||||
if ($this->getOption('aCookie') !== null) {
|
||||
$username = $this->loadCacheValue('username');
|
||||
if ($username !== null) {
|
||||
if ($username) {
|
||||
$name = $username . '\'s ' . parent::getName();
|
||||
}
|
||||
}
|
||||
|
@ -34,8 +34,7 @@ class FurAffinityUserBridge extends BridgeAbstract
|
||||
|
||||
$url = self::URI . '/gallery/' . $this->getInput('searchUsername');
|
||||
|
||||
$html = getSimpleHTMLDOM($url, [], $opt)
|
||||
or returnServerError('Could not load the user\'s gallery page.');
|
||||
$html = getSimpleHTMLDOM($url, [], $opt);
|
||||
|
||||
$submissions = $html->find('section[id=gallery-gallery]', 0)->find('figure');
|
||||
foreach ($submissions as $submission) {
|
||||
|
@ -31,7 +31,7 @@ class GBAtempBridge extends BridgeAbstract
|
||||
$img = $this->findItemImage($newsItem, 'a.news_image');
|
||||
$time = $this->findItemDate($newsItem);
|
||||
$author = $newsItem->find('a.username', 0)->plaintext;
|
||||
$title = $this->decodeHtmlEntities($newsItem->find('h3.news_title', 0)->plaintext);
|
||||
$title = $this->decodeHtmlEntities($newsItem->find('h2.news_title', 0)->plaintext);
|
||||
$content = $this->fetchPostContent($url, self::URI);
|
||||
$this->items[] = $this->buildItem($url, $title, $author, $time, $img, $content);
|
||||
unset($newsItem); // Some items are heavy, freeing the item proactively helps saving memory
|
||||
@ -41,7 +41,7 @@ class GBAtempBridge extends BridgeAbstract
|
||||
foreach ($html->find('li.portal_review') as $reviewItem) {
|
||||
$url = urljoin(self::URI, $reviewItem->find('a.review_boxart', 0)->href);
|
||||
$img = $this->findItemImage($reviewItem, 'a.review_boxart');
|
||||
$title = $this->decodeHtmlEntities($reviewItem->find('h2.review_title', 0)->plaintext);
|
||||
$title = $this->decodeHtmlEntities($reviewItem->find('div.review_title', 0)->find('h2', 0)->plaintext);
|
||||
$content = getSimpleHTMLDOMCached($url);
|
||||
$author = $content->find('span.author--name', 0)->plaintext;
|
||||
$time = $this->findItemDate($content);
|
||||
|
164
bridges/GULPProjekteBridge.php
Normal file
164
bridges/GULPProjekteBridge.php
Normal file
@ -0,0 +1,164 @@
|
||||
<?php
|
||||
|
||||
use Facebook\WebDriver\Exception\NoSuchElementException;
|
||||
use Facebook\WebDriver\Remote\RemoteWebElement;
|
||||
use Facebook\WebDriver\WebDriverBy;
|
||||
use Facebook\WebDriver\WebDriverExpectedCondition;
|
||||
|
||||
class GULPProjekteBridge extends WebDriverAbstract
|
||||
{
|
||||
const NAME = 'GULP Projekte';
|
||||
const URI = 'https://www.gulp.de/gulp2/g/projekte';
|
||||
const DESCRIPTION = 'Projektsuche';
|
||||
const MAINTAINER = 'hleskien';
|
||||
|
||||
const MAXITEMS = 60;
|
||||
|
||||
/**
|
||||
* Adds accept language german to the Chrome Options.
|
||||
*
|
||||
* @return Facebook\WebDriver\Chrome\ChromeOptions
|
||||
*/
|
||||
protected function getBrowserOptions()
|
||||
{
|
||||
$chromeOptions = parent::getBrowserOptions();
|
||||
$chromeOptions->addArguments(['--accept-lang=de']);
|
||||
return $chromeOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Facebook\WebDriver\Exception\NoSuchElementException
|
||||
* @throws Facebook\WebDriver\Exception\TimeoutException
|
||||
*/
|
||||
protected function clickAwayCookieBanner()
|
||||
{
|
||||
$this->getDriver()->wait()->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::id('onetrust-reject-all-handler')));
|
||||
$buttonRejectCookies = $this->getDriver()->findElement(WebDriverBy::id('onetrust-reject-all-handler'));
|
||||
$buttonRejectCookies->click();
|
||||
$this->getDriver()->wait()->until(WebDriverExpectedCondition::invisibilityOfElementLocated(WebDriverBy::id('onetrust-reject-all-handler')));
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws Facebook\WebDriver\Exception\NoSuchElementException
|
||||
* @throws Facebook\WebDriver\Exception\TimeoutException
|
||||
*/
|
||||
protected function clickNextPage()
|
||||
{
|
||||
$nextPage = $this->getDriver()->findElement(WebDriverBy::xpath('//app-linkable-paginator//li[@id="next-page"]/a'));
|
||||
$href = $nextPage->getAttribute('href');
|
||||
$nextPage->click();
|
||||
$this->getDriver()->wait()->until(WebDriverExpectedCondition::not(
|
||||
WebDriverExpectedCondition::presenceOfElementLocated(
|
||||
WebDriverBy::xpath('//app-linkable-paginator//li[@id="next-page"]/a[@href="' . $href . '"]')
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the uri of the 'Projektanbieter' logo or false if there is
|
||||
* no logo present in the item.
|
||||
*
|
||||
* @return string | false
|
||||
*/
|
||||
protected function getLogo(RemoteWebElement $item)
|
||||
{
|
||||
try {
|
||||
$logo = $item->findElement(WebDriverBy::tagName('img'))->getAttribute('src');
|
||||
if (str_starts_with($logo, 'http')) {
|
||||
// different domain
|
||||
return $logo;
|
||||
} else {
|
||||
// relative path
|
||||
$remove = substr(self::URI, strrpos(self::URI, '/') + 1);
|
||||
return substr(self::URI, 0, -strlen($remove)) . $logo;
|
||||
}
|
||||
} catch (NoSuchElementException $e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a string like "vor einigen Minuten" into a reasonable timestamp.
|
||||
* Long and complicated, but we don't want to be more specific than
|
||||
* the information we have available.
|
||||
*
|
||||
* @throws Exception If the DateInterval can't be parsed.
|
||||
*/
|
||||
protected function getTimestamp(string $timeAgo): int
|
||||
{
|
||||
$dateTime = new DateTime();
|
||||
$dateArray = explode(' ', $dateTime->format('Y m d H i s'));
|
||||
$quantityStr = explode(' ', $timeAgo)[1];
|
||||
// convert possible word into a number
|
||||
if (in_array($quantityStr, ['einem', 'einer', 'einigen'])) {
|
||||
$quantity = 1;
|
||||
} else {
|
||||
$quantity = intval($quantityStr);
|
||||
}
|
||||
// subtract time ago + inferior units for lower precision
|
||||
if (str_contains($timeAgo, 'Sekunde')) {
|
||||
$interval = new DateInterval('PT' . $quantity . 'S');
|
||||
} elseif (str_contains($timeAgo, 'Minute')) {
|
||||
$interval = new DateInterval('PT' . $quantity . 'M' . $dateArray[5] . 'S');
|
||||
} elseif (str_contains($timeAgo, 'Stunde')) {
|
||||
$interval = new DateInterval('PT' . $quantity . 'H' . $dateArray[4] . 'M' . $dateArray[5] . 'S');
|
||||
} elseif (str_contains($timeAgo, 'Tag')) {
|
||||
$interval = new DateInterval('P' . $quantity . 'DT' . $dateArray[3] . 'H' . $dateArray[4] . 'M' . $dateArray[5] . 'S');
|
||||
} else {
|
||||
throw new UnexpectedValueException($timeAgo);
|
||||
}
|
||||
$dateTime = $dateTime->sub($interval);
|
||||
return $dateTime->getTimestamp();
|
||||
}
|
||||
|
||||
/**
|
||||
* The main loop which clicks through search result pages and puts
|
||||
* the content into the $items array.
|
||||
*
|
||||
* @throws Facebook\WebDriver\Exception\NoSuchElementException
|
||||
* @throws Facebook\WebDriver\Exception\TimeoutException
|
||||
*/
|
||||
public function collectData()
|
||||
{
|
||||
parent::collectData();
|
||||
|
||||
try {
|
||||
$this->clickAwayCookieBanner();
|
||||
$this->setIcon($this->getDriver()->findElement(WebDriverBy::xpath('//link[@rel="shortcut icon"]'))->getAttribute('href'));
|
||||
|
||||
while (true) {
|
||||
$items = $this->getDriver()->findElements(WebDriverBy::tagName('app-project-view'));
|
||||
foreach ($items as $item) {
|
||||
$feedItem = [];
|
||||
|
||||
$heading = $item->findElement(WebDriverBy::xpath('.//app-heading-tag/h1/a'));
|
||||
$feedItem['title'] = $heading->getText();
|
||||
$feedItem['uri'] = 'https://www.gulp.de' . $heading->getAttribute('href');
|
||||
$info = $item->findElement(WebDriverBy::tagName('app-icon-info-list'));
|
||||
if ($logo = $this->getLogo($item)) {
|
||||
$feedItem['enclosures'] = [$logo];
|
||||
}
|
||||
if (str_contains($info->getText(), 'Projektanbieter:')) {
|
||||
$feedItem['author'] = $info->findElement(WebDriverBy::xpath('.//li/span[2]/span'))->getText();
|
||||
} else {
|
||||
// mostly "Direkt vom Auftraggeber" or "GULP Agentur"
|
||||
$feedItem['author'] = $item->findElement(WebDriverBy::tagName('b'))->getText();
|
||||
}
|
||||
$feedItem['content'] = $item->findElement(WebDriverBy::xpath('.//p[@class="description"]'))->getText();
|
||||
$timeAgo = $item->findElement(WebDriverBy::xpath('.//small[contains(@class, "time-ago")]'))->getText();
|
||||
$feedItem['timestamp'] = $this->getTimestamp($timeAgo);
|
||||
|
||||
$this->items[] = $feedItem;
|
||||
}
|
||||
|
||||
if (count($this->items) < self::MAXITEMS) {
|
||||
$this->clickNextPage();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$this->cleanUp();
|
||||
}
|
||||
}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user