diff --git a/README.md b/README.md
index d4591fb..9db1574 100644
--- a/README.md
+++ b/README.md
@@ -6,9 +6,10 @@
[![NuGet download count](https://img.shields.io/nuget/dt/SharpGrabber)](https://www.nuget.org/packages/SharpGrabber)
This repository contains multiple related projects:
-- `SharpGrabber` is a *.NET Standard* library for scraping top media providers and grabbing high quality video, audio and information.
-- `SharpGrabber.Converter` is a *.NET Standard* library based on `ffmpeg` shared libraries to join audio and video streams. This is particularly useful when grabbing high quality *YouTube* media that might be separated into audio and video files. It is also used for merging HLS stream segments.
-- `SharpGrabber.Desktop` A cross-platform desktop application which utilizes both mentioned libraries to expose their functionality to desktop end-users.
+- `SharpGrabber` is a *.NET Standard* library for scraping top media providers and grabbing high quality video, audio and information.
+- `SharpGrabber.Converter` is a *.NET Standard* library based on `ffmpeg` shared libraries to join audio and video streams. This is particularly useful when grabbing high quality *YouTube* media that might be separated into audio and video files. It is also used for merging HLS stream segments.
+- `SharpGrabber.BlackWidow` is a *.NET Standard* library for grabbing with JavaScript, which has many advantages over using scattered NuGet packages.
+- `SharpGrabber.Desktop` A cross-platform desktop application which utilizes all three libraries mentioned above to expose their functionality to desktop end-users.
# How to Use
**⭐ Please give a star if you find this project useful!**
@@ -24,7 +25,7 @@ This repository contains multiple related projects:
The `SharpGrabber` package defines abstractions only. The actual grabbers have their own packages and should be installed separately.
### SharpGrabber - Core Package
- Install-Package SharpGrabber -Version 2.0.2
+ Install-Package SharpGrabber -Version 2.1
### SharpGrabber.Converter
It's an optional package to work with media files. Using this package, you can easily concatenate video segments, or mux audio and video channels.
@@ -95,9 +96,10 @@ The good news is no functionality has been removed, so with a minor refactoring,
I strongly recommend that you upgrade, v2 has a much cleaner structure and code.
-
-## SharpGrabber.Desktop 3.3
-- It uses every package mentioned above and supports all of the mentioned providers!
+
+## SharpGrabber.Desktop
+### Version 3.3
+- Grabs from every source supported by official grabbers.
- Displays information and downloads videos, audios, images etc.
- Merges YouTube separated audio and video streams into complete media files. It can join HLS segments as well!
@@ -111,12 +113,29 @@ Requirements of the cross-platform desktop application to run and operate correc
+# Introducing BlackWidow
+
+
+BlackWidow executes scripts written specifically for grabbing, rather than relying on .NET assemblies.
+- **Always Up-to-date:** The scripts are always kept up-to-date at runtime; so the functionality of the host application won't break as the sources change - at least not for long!
+- **ECMAScript Support:** Supports JavaScript/ECMAScript out of the box.
+- **Easy Maintenance:** *JavaScript* is darn easy to write and understand! This helps contributors to quickly write new grabbers or fix the existing ones.
+- **Secure**: The scripts are executed in a sandbox environment, and they only have access to what the BlackWidow API exposes to them.
+- **Highly Customizable:** Almost everything is open for extension or replacement. Make new script interpreters, custom grabber repositories, or roll out your own interpreter APIs
+Read more + Documentation
+
## Contribution
You are most welcome to contribute!
-- Support for more media providers such as *DailyMotion*, *Instagram*, *Facebook*, *Twitch* etc.
+- Authentication mechanisms for grabbers e.g. Instagram Login
+- Support for more media providers such as *DailyMotion*, *Facebook*, *Twitch* etc.
- Accelerate downloads in the desktop app (like a download manager)
+## Disclaimer
+SharpGrabber library, BlackWidow and other projects and libraries provided in this repository are developed for educational purposes.
+Since it's illegal to extract copyrighted data, you should make sure your usage of the tools provided here complies with copyright laws.
+Contributors to these tools are not responsible for any copyright infringement that may occur per usage.
+
## License
Copyright © 2021 Javid Shoaei and other contributors
diff --git a/SharpGrabber.sln b/SharpGrabber.sln
index 56a9024..f3cf591 100644
--- a/SharpGrabber.sln
+++ b/SharpGrabber.sln
@@ -24,6 +24,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SharpGrabber.Hls", "src\Sha
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SharpGrabber.Instagram", "src\SharpGrabber.Instagram\SharpGrabber.Instagram.csproj", "{094B729B-9871-4A2C-9228-9AAEE66F135D}"
EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SharpGrabber.BlackWidow", "src\SharpGrabber.BlackWidow\SharpGrabber.BlackWidow.csproj", "{9F3A8C86-8F28-4F54-B8A6-DBB49DDB5171}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SharpGrabber.BlackWidow.Tests", "tests\SharpGrabber.BlackWidow.Tests\SharpGrabber.BlackWidow.Tests.csproj", "{4CB41014-D036-4090-B6FA-4CFB01D82C3A}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{ADFEEE61-D79B-4F91-A192-F6A2E949673C}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -62,10 +68,21 @@ Global
{094B729B-9871-4A2C-9228-9AAEE66F135D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{094B729B-9871-4A2C-9228-9AAEE66F135D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{094B729B-9871-4A2C-9228-9AAEE66F135D}.Release|Any CPU.Build.0 = Release|Any CPU
+ {9F3A8C86-8F28-4F54-B8A6-DBB49DDB5171}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {9F3A8C86-8F28-4F54-B8A6-DBB49DDB5171}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {9F3A8C86-8F28-4F54-B8A6-DBB49DDB5171}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {9F3A8C86-8F28-4F54-B8A6-DBB49DDB5171}.Release|Any CPU.Build.0 = Release|Any CPU
+ {4CB41014-D036-4090-B6FA-4CFB01D82C3A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {4CB41014-D036-4090-B6FA-4CFB01D82C3A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {4CB41014-D036-4090-B6FA-4CFB01D82C3A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {4CB41014-D036-4090-B6FA-4CFB01D82C3A}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
+ GlobalSection(NestedProjects) = preSolution
+ {4CB41014-D036-4090-B6FA-4CFB01D82C3A} = {ADFEEE61-D79B-4F91-A192-F6A2E949673C}
+ EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {0003E70E-C9A2-459C-A6A0-540449AC7A87}
EndGlobalSection
diff --git a/assets/blackwidow-logo-text-sm.png b/assets/blackwidow-logo-text-sm.png
new file mode 100644
index 0000000..36c3c63
Binary files /dev/null and b/assets/blackwidow-logo-text-sm.png differ
diff --git a/assets/blackwidow-logo-text.png b/assets/blackwidow-logo-text.png
new file mode 100644
index 0000000..6be1754
Binary files /dev/null and b/assets/blackwidow-logo-text.png differ
diff --git a/assets/blackwidow-logo-text.psd b/assets/blackwidow-logo-text.psd
new file mode 100644
index 0000000..39ef85e
Binary files /dev/null and b/assets/blackwidow-logo-text.psd differ
diff --git a/assets/blackwidow-logo.png b/assets/blackwidow-logo.png
new file mode 100644
index 0000000..b93e412
Binary files /dev/null and b/assets/blackwidow-logo.png differ
diff --git a/assets/blackwidow-logo.psd b/assets/blackwidow-logo.psd
new file mode 100644
index 0000000..69ef5da
Binary files /dev/null and b/assets/blackwidow-logo.psd differ
diff --git a/blackwidow/README.md b/blackwidow/README.md
new file mode 100644
index 0000000..ebbfd9a
--- /dev/null
+++ b/blackwidow/README.md
@@ -0,0 +1,27 @@
+
+
+# BlackWidow
+
+BlackWidow is a .NET library based on SharpGrabber. Rather than relying on .NET assemblies, BlackWidow executes scripts written specifically for grabbing.
+
+## Why use BlackWidow?
+BlackWidow gives you the following advantages over the traditional NuGet package approach:
+
+- **Always Up-to-date:** The scripts are always kept up-to-date at runtime; so the functionality of the host application won't break as the sources change - at least not for long!
+- **ECMAScript Support:** Supports JavaScript/ECMAScript out of the box.
+- **Easy Maintenance:** *JavaScript* is darn easy to write and understand! This helps contributors to quickly write new grabbers or fix the existing ones.
+- **Secure**: The scripts are executed in a sandbox environment, and they only have access to what the BlackWidow API exposes to them.
+- **Highly Customizable:** Almost everything is open for extension or replacement. Make new script interpreters, custom grabber repositories, or roll out your own interpreter APIs
+
+## How does it work?
+
+BlackWidow keeps a collection of scripts locally - called the local repository.
+Each script gets interpreted as an object implementing `IGrabber`.
+To keep the scripts up-to-date, a remote repository is constantly monitored as the single source of truth.
+
+*TODO:* Read the Documentation
+
+# Installation
+*WIP*
+
+<- Back to Home Page
diff --git a/blackwidow/repo/feed.json b/blackwidow/repo/feed.json
new file mode 100644
index 0000000..4fdf90b
--- /dev/null
+++ b/blackwidow/repo/feed.json
@@ -0,0 +1,22 @@
+{
+ "scripts": [
+ {
+ "id": "vimeo.com",
+ "name": "Vimeo",
+ "version": "1.0",
+ "type": "JavaScript",
+ "apiVersion": 1,
+ "supportedRegularExpressions": [ "^https?://(www\\.|player\\.)?vimeo\\.com/(video/)?([0-9]+)" ],
+ "file": "scripts/vimeo.js"
+ },
+ {
+ "id": "pornhub.com",
+ "name": "PornHub",
+ "version": "1.0",
+ "type": "JavaScript",
+ "apiVersion": 1,
+ "supportedRegularExpressions": [ "^(https?:\\/\\/)?(www\\.)?pornhub\\.com\\/([^\\/]+)viewkey=(\\w+).*$" ],
+ "file": "scripts/pornhub.js"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/blackwidow/repo/scripts/pornhub.js b/blackwidow/repo/scripts/pornhub.js
new file mode 100644
index 0000000..26b7ef1
--- /dev/null
+++ b/blackwidow/repo/scripts/pornhub.js
@@ -0,0 +1,115 @@
+const urlMatcher = /^(https?:\/\/)?(www\.)?pornhub\.com\/([^\/]+)viewkey=(\w+).*$/i
+const flashVarsFinder = /^\s*(var|let)\s+(flashvars[\w_]+)\s+=/mi
+
+const getViewId = uri => {
+ const url = new URL(uri)
+ const match = urlMatcher.exec(uri)
+ if (!match)
+ return undefined
+ return match[4]
+}
+
+const getStdUrl = url => {
+ return `https://www.pornhub.com/view_video.php?viewkey=${url}`
+}
+
+const parseFlashVarsScript = doc => {
+ let source
+ let varName
+ doc.selectAll('script').forEach(elem => {
+ const match = flashVarsFinder.exec(elem.innerText)
+ if (match) {
+ source = elem.innerText
+ varName = match[2]
+ }
+ })
+
+ const flashVars = new Function('let playerObjList = {};'+source + ';return '+varName+';')()
+ if (!flashVars)
+ throw new GrabException('Could not extract flashVars.')
+ return flashVars
+}
+
+const updateResult = (result, vars) => {
+ const parseBool = str => typeof str === 'boolean' ? str : new Function('return ' + str)();
+
+ if (parseBool(vars.video_unavailable))
+ throw new GrabException('This video is unavailable.')
+ if (parseBool(vars.video_unavailable_country))
+ throw new GrabException('This video is unavailable in your country.')
+
+ const duration = vars.video_duration * 1000 // milliseconds
+
+ result.title = vars.video_title
+
+ result.grab('info', {
+ length: duration
+ })
+
+ result.grab('image', {
+ resourceUri: vars.image_url,
+ type: 'primary'
+ })
+
+ vars.mediaDefinitions.forEach(def => {
+ if (!def.quality || def.remote || !def.videoUrl)
+ return
+
+ if (def.format === 'hls') {
+ // grab HLS stream
+ if (Array.isArray(def.quality)) {
+ result.grab('hlsStreamReference', {
+ resourceUri: def.videoUrl,
+ playlistType: 'master',
+ resolution: def.quality.join(',')
+ })
+ } else {
+ result.grab('hlsStreamReference', {
+ resourceUri: def.videoUrl,
+ playlistType: 'stream',
+ resolution: def.quality
+ })
+ }
+ } else {
+ // grab mp4 video
+ result.grab('media', {
+ resourceUri: def.videoUrl,
+ format: {
+ mime: 'video/mp4',
+ extension: 'mp4',
+ channels: 'both',
+ length: duration,
+ container: 'mp4,'
+ resolution: def.quality,
+ formatTitle: 'MP4 ' + def.quality,
+ }
+ })
+ }
+ })
+}
+
+grabber.supports = uri => {
+ return getViewId(uri) !== undefined
+}
+
+grabber.grab = (request, result) => {
+
+ // init
+ const viewId = getViewId(request.url)
+ if (!viewId)
+ return false
+
+ // download page
+ const url = getStdUrl(viewId)
+ const response = http.client.get({
+ url
+ })
+ response.assertSuccess()
+
+ // parse response HTML
+ const doc = html.parse(response.bodyText)
+ const flashVars = parseFlashVarsScript(doc)
+ updateResult(result, flashVars)
+
+ return true
+}
diff --git a/blackwidow/repo/scripts/vimeo.js b/blackwidow/repo/scripts/vimeo.js
new file mode 100644
index 0000000..58664cb
--- /dev/null
+++ b/blackwidow/repo/scripts/vimeo.js
@@ -0,0 +1,83 @@
+const urlRegex = /^https?:\/\/(www\.|player\.)?vimeo\.com\/(video\/)?([0-9]+)/
+
+function getVideoId(url) {
+ const match = urlRegex.exec(url)
+ return match ? match[3] : undefined
+}
+
+function getConfigUrl(videoId) {
+ return 'https://player.vimeo.com/video/{0}/config'.replace('{0}', videoId)
+}
+
+function fetchConfig(videoId) {
+ const url = getConfigUrl(videoId)
+ const response = http.client.get({
+ url,
+ expectText: true
+ })
+ response.assertSuccess()
+ return JSON.parse(response.bodyText)
+}
+
+function setGrabResult(result, config) {
+ if (!config.request.files)
+ throw new GrabException('Video is unavailable.')
+
+ // add info
+ result.title = config.video.title
+ result.grab('info', {
+ author: config.video.owner?.name,
+ length: config.video.duration * 1000,
+ })
+
+ // add images
+ if (config.video.thumbs) {
+ for (var key in config.video.thumbs) {
+ const isBase = Number.isNaN(Number(key))
+ const size = isBase ? undefined : {
+ width: key,
+ height: key * 0.5625
+ };
+ result.grab('image', {
+ resourceUri: config.video.thumbs[key],
+ type: isBase ? 'primary' : 'thumbnail',
+ size
+ })
+ }
+ }
+
+ // add media
+ config.request.files.progressive.forEach(file => {
+ const fileMime = file.mime || 'video/mp4'
+ const fileExt = mime.getExtension(fileMime)
+ const containerName = fileExt.toUpperCase()
+ result.grab('media', {
+ resourceUri: file.url,
+ channels: 'both',
+ container: containerName,
+ resolution: file.quality,
+ formatTitle: containerName + ' ' + file.quality,
+ pixelWidth: file.width,
+ pixelHeight: file.height,
+ format: {
+ mime: fileMime,
+ extension: fileExt
+ }
+ })
+ })
+}
+
+grabber.supports = url => Boolean(getVideoId(url))
+
+grabber.grab = (request, result) => {
+ const videoId = getVideoId(request.url)
+ if (!videoId)
+ return false
+
+ const config = fetchConfig(videoId)
+ if (!config)
+ throw new GrabException('Failed to fetch video config.')
+
+ setGrabResult(result, config)
+ return true
+}
\ No newline at end of file
diff --git a/blackwidow/schema/feed.json b/blackwidow/schema/feed.json
new file mode 100644
index 0000000..d1baca6
--- /dev/null
+++ b/blackwidow/schema/feed.json
@@ -0,0 +1,57 @@
+{
+ "$schema": "https://json-schema.org/draft-04/schema",
+ "$id": "https://raw.githubusercontent.com/dotnettools/SharpGrabber/blackwidow/blackwidow/schema/feed.json",
+ "title": "Feed",
+ "description": "BlackWidow Grabber Repository Feed",
+ "type": "object",
+ "properties": {
+ "scripts": {
+ "description": "Array of scripts defined in this feed",
+ "type": "array",
+ "items": {
+ "description": "BlackWidow Grabber Repository Script",
+ "type": "object",
+ "properties": {
+ "id": {
+ "description": "The unique identifier for the script",
+ "type": "string"
+ },
+ "name": {
+ "description": "A friendly name for the script",
+ "type": "string"
+ },
+ "version": {
+ "description": "The semantic version of the script",
+ "type": "string",
+ "minLength": 5,
+ "maxLength": 14,
+ "pattern": "^(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)$"
+ },
+ "type": {
+ "description": "Type of the script",
+ "type": "string",
+ "enum": [ "JavaScript" ]
+ },
+ "apiVersion": {
+ "description": "BlackWidow API version",
+ "type": "integer",
+ "minimum": 1
+ },
+ "supportedRegularExpressions": {
+ "description": "Array of regular expressions the script can potentially support",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "file": {
+ "description": "Virtual path to the script file, relative to the feed",
+ "type": "string"
+ },
+ },
+ "required": [ "id", "name", "version", "type", "apiVersion", "supportedRegularExpressions", "file" ]
+ }
+ }
+ },
+ "required": [ "scripts" ]
+}
\ No newline at end of file
diff --git a/src/SharpGrabber.Adult/PornHubGrabber.cs b/src/SharpGrabber.Adult/PornHubGrabber.cs
index 68ba74c..455b19d 100644
--- a/src/SharpGrabber.Adult/PornHubGrabber.cs
+++ b/src/SharpGrabber.Adult/PornHubGrabber.cs
@@ -123,7 +123,7 @@ protected virtual void Grab(GrabResult result, List resources, JObject
if (options.Flags.HasFlag(GrabOptionFlags.GrabImages))
{
var image_url = new Uri(result.OriginalUri, flashVars.SelectToken("$.image_url").Value());
- resources.Add(new GrabbedImage(GrabbedImageType.Primary, null, image_url));
+ resources.Add(new GrabbedImage(GrabbedImageType.Primary, image_url));
}
result.Title = flashVars.SelectToken("$.video_title").Value();
@@ -140,7 +140,7 @@ protected virtual void Grab(GrabResult result, List resources, JObject
var url = quality.Value("url");
if (string.IsNullOrEmpty(url))
continue;
- var vid = new GrabbedMedia(new Uri(result.OriginalUri, url), result.OriginalUri, DefaultMediaFormat, MediaChannels.Both);
+ var vid = new GrabbedMedia(new Uri(result.OriginalUri, url), DefaultMediaFormat, MediaChannels.Both);
vid.Resolution = quality.Value("text");
var qint = StringHelper.ForceParseInt(vid.Resolution);
grabbed.Add(qint, vid);
@@ -166,7 +166,7 @@ protected virtual void Grab(GrabResult result, List resources, JObject
switch (format.ToLowerInvariant())
{
case "mp4":
- var m = new GrabbedMedia(uri, result.OriginalUri, DefaultMediaFormat, MediaChannels.Both)
+ var m = new GrabbedMedia(uri, DefaultMediaFormat, MediaChannels.Both)
{
Resolution = resol,
FormatTitle = $"MP4 {resol}",
@@ -174,7 +174,7 @@ protected virtual void Grab(GrabResult result, List resources, JObject
grabbed.Add(quality, m);
break;
case "hls":
- var sr = new GrabbedStreamReference(uri, result.OriginalUri)
+ var sr = new GrabbedHlsStreamReference(uri)
{
Resolution = resol,
PlaylistType = playlistType,
diff --git a/src/SharpGrabber.Adult/XnxxGrabber.cs b/src/SharpGrabber.Adult/XnxxGrabber.cs
index 5fd3f59..323e5bc 100644
--- a/src/SharpGrabber.Adult/XnxxGrabber.cs
+++ b/src/SharpGrabber.Adult/XnxxGrabber.cs
@@ -49,10 +49,10 @@ protected override async Task InternalGrabAsync(Uri uri, Cancellatio
// grab images
var img = (paramMap.GetOrDefault("image") ?? paramMap.GetOrDefault("ThumbUrl169") ?? paramMap.GetOrDefault("ThumbUrl")) as string;
if (Uri.TryCreate(img, UriKind.Absolute, out var imgUri))
- resources.Add(new GrabbedImage(GrabbedImageType.Thumbnail, uri, imgUri));
+ resources.Add(new GrabbedImage(GrabbedImageType.Thumbnail, imgUri));
img = (paramMap.GetOrDefault("ThumbSlideBig") ?? paramMap.GetOrDefault("ThumbSlide")) as string;
if (Uri.TryCreate(img, UriKind.Absolute, out imgUri))
- resources.Add(new GrabbedImage(GrabbedImageType.Preview, uri, imgUri));
+ resources.Add(new GrabbedImage(GrabbedImageType.Preview, imgUri));
// grab resources
var hls = paramMap["VideoHLS"] as string;
diff --git a/src/SharpGrabber.BlackWidow/BlackWidowConstants.cs b/src/SharpGrabber.BlackWidow/BlackWidowConstants.cs
new file mode 100644
index 0000000..e6843dd
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/BlackWidowConstants.cs
@@ -0,0 +1,38 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Defines BlackWidow-related constants.
+ ///
+ public static class BlackWidowConstants
+ {
+ public static class GitHub
+ {
+ public static class OfficialRepository
+ {
+ ///
+ /// The offical repository name
+ ///
+ public const string RepositoryAddress = "dotnettools/SharpGrabber";
+
+ ///
+ /// Name of the main branch
+ ///
+ public const string MasterBranch = "master";
+
+ ///
+ /// Path to the directory that contains the feed file and the scripts
+ ///
+ public const string RootPath = "blackwidow/repo";
+
+ ///
+ /// Name of the feed JSON file
+ ///
+ public const string FeedFileName = "feed.json";
+ }
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/BlackWidowInitializer.cs b/src/SharpGrabber.BlackWidow/BlackWidowInitializer.cs
new file mode 100644
index 0000000..aaa85e3
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/BlackWidowInitializer.cs
@@ -0,0 +1,32 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ internal static class BlackWidowInitializer
+ {
+ static BlackWidowInitializer()
+ {
+ EnsureLoaded(Hls.HlsGrabber.Initializer);
+ }
+
+ public static void Test()
+ {
+ // nothing should be done here,
+ // the static constructor would run once.
+ }
+
+ private static void EnsureLoaded(params Type[] types)
+ {
+ foreach (var type in types)
+ {
+ // create a dummy instance just to ensure the type is loaded
+ var o = Activator.CreateInstance(type);
+ if (o is IDisposable disposable)
+ disposable.Dispose();
+ }
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/BlackWidowService.cs b/src/SharpGrabber.BlackWidow/BlackWidowService.cs
new file mode 100644
index 0000000..758f418
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/BlackWidowService.cs
@@ -0,0 +1,289 @@
+using DotNetTools.SharpGrabber.BlackWidow.Exceptions;
+using DotNetTools.SharpGrabber.BlackWidow.Host;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter;
+using DotNetTools.SharpGrabber.BlackWidow.Repository;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using DotNetTools.SharpGrabber.BlackWidow.Definitions;
+using DotNetTools.SharpGrabber.BlackWidow.Internal;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Default implementation for
+ ///
+ public class BlackWidowService : IBlackWidowService
+ {
+ private readonly ConcurrentDictionary _grabbers =
+ new(StringComparer.InvariantCultureIgnoreCase);
+ private readonly BlackWidowGrabber _grabber;
+
+ private readonly IGrabberRepositoryChangeDetector _changeDetector;
+ private readonly ConcurrentHashSet _scriptsUsed = new();
+ private readonly ConcurrentHashSet _scriptsUpdating = new();
+ private IGrabberRepositoryFeed _localFeed;
+ private IGrabberRepositoryFeed _remoteFeed;
+
+ protected BlackWidowService(IGrabberRepository localRepository, IGrabberRepository remoteRepository,
+ IGrabberServices grabberServices,
+ IScriptHost scriptHost, IGrabberScriptInterpreterService interpreterService, IGrabberRepositoryChangeDetector changeDetector)
+ {
+ _changeDetector = changeDetector;
+ Interpreters = interpreterService ?? throw new ArgumentNullException(nameof(interpreterService));
+ LocalRepository = localRepository ?? throw new ArgumentNullException(nameof(localRepository));
+ RemoteRepository = remoteRepository ?? throw new ArgumentNullException(nameof(remoteRepository));
+ ScriptHost = scriptHost;
+ changeDetector.RepositoryChanged += ChangeDetector_RepositoryChanged;
+ _grabber = new BlackWidowGrabber(this, grabberServices ?? throw new ArgumentNullException(nameof(grabberServices)));
+ }
+
+ public IScriptHost ScriptHost { get; }
+
+ ///
+ /// Gets the interpreter service.
+ ///
+ public IGrabberScriptInterpreterService Interpreters { get; }
+
+ public IGrabberRepository LocalRepository { get; }
+
+ public IGrabberRepository RemoteRepository { get; }
+
+ public IBlackWidowGrabber Grabber => _grabber;
+
+ ///
+ /// Creates a new instance of .
+ ///
+ public static async Task CreateAsync(IGrabberRepository localRepository,
+ IGrabberRepository remoteRepository,
+ IGrabberServices grabberServices,
+ IScriptHost scriptHost, IGrabberScriptInterpreterService interpreterService = null,
+ IGrabberRepositoryChangeDetector changeDetector = null)
+ {
+ interpreterService ??= new GrabberScriptInterpreterService();
+ changeDetector ??= new GrabberRepositoryChangeDetector(new[] { localRepository, remoteRepository });
+ var service = new BlackWidowService(localRepository, remoteRepository, grabberServices, scriptHost, interpreterService, changeDetector);
+ await service.LoadLocalFeedAsync().ConfigureAwait(false);
+ return service;
+ }
+
+ public IEnumerable GetLocalCandidates(Uri uri)
+ {
+ return _grabbers.Values.Where(g => g.Supports(uri));
+ }
+
+ public IGrabber GetLocalScript(string scriptId)
+ => _grabbers.GetOrDefault(scriptId);
+
+ public IEnumerable GetRemoteCandidates(Uri uri)
+ {
+ if (_remoteFeed == null)
+ return Enumerable.Empty();
+
+ return _remoteFeed
+ .GetScripts()
+ .Where(s => s.IsMatch(uri));
+ }
+
+ public async Task GetScriptAsync(string scriptId)
+ {
+ // init
+ var localInfo = _localFeed.GetScript(scriptId);
+ var remoteInfo = _remoteFeed?.GetScript(scriptId);
+ if (localInfo == null)
+ {
+ await LoadLocalFeedAsync().ConfigureAwait(false);
+ localInfo = _localFeed.GetScript(scriptId);
+ }
+
+ var updateNeeded = localInfo == null ||
+ (remoteInfo != null && remoteInfo.GetVersion() > localInfo.GetVersion());
+
+ if (localInfo == null && remoteInfo == null)
+ return null;
+
+ // fetch the script
+ if (updateNeeded)
+ {
+ var source = await RemoteRepository.FetchSourceAsync(remoteInfo).ConfigureAwait(false);
+ await LocalRepository.PutAsync(remoteInfo, source).ConfigureAwait(false);
+ _grabbers.TryRemove(scriptId, out _);
+ await LoadLocalFeedAsync().ConfigureAwait(false);
+ }
+
+ // get local grabber
+ return _grabbers.GetOrDefault(scriptId);
+ }
+
+ public async Task UpdateFeedAsync()
+ {
+ _remoteFeed = await RemoteRepository.GetFeedAsync().ConfigureAwait(false);
+ }
+
+ public void Dispose()
+ {
+ _changeDetector?.Dispose();
+ }
+
+ private async Task LoadLocalFeedAsync()
+ {
+ _localFeed = await LocalRepository.GetFeedAsync().ConfigureAwait(false);
+ await LoadLocalGrabbers().ConfigureAwait(false);
+ }
+
+ private async Task LoadLocalGrabbers()
+ {
+ foreach (var scriptInfo in _localFeed.GetScripts())
+ {
+ if (_grabbers.ContainsKey(scriptInfo.Id))
+ continue;
+ var scriptSource = await LocalRepository.FetchSourceAsync(scriptInfo).ConfigureAwait(false);
+ await LoadGrabberAsync(scriptInfo, scriptSource).ConfigureAwait(false);
+ }
+ }
+
+ private async Task LoadGrabberAsync(IGrabberRepositoryScript scriptInfo,
+ IGrabberScriptSource scriptSource)
+ {
+ var interpreter = Interpreters.GetInterpreter(scriptInfo.Type);
+ if (interpreter == null)
+ throw new ScriptInterpretException($"No interpreter is registered for {scriptInfo.Type}.");
+
+ var grabber = await interpreter.InterpretAsync(scriptInfo, scriptSource, scriptInfo.ApiVersion)
+ .ConfigureAwait(false);
+ _grabbers.TryAdd(scriptInfo.Id, grabber);
+ return grabber;
+ }
+
+ private void ChangeDetector_RepositoryChanged(IGrabberRepository repository, IGrabberRepositoryFeed feed, IGrabberRepositoryFeed prevFeed)
+ {
+ if (repository != LocalRepository && repository != RemoteRepository)
+ return;
+
+ var isLocal = LocalRepository == repository;
+ if (isLocal)
+ _localFeed = feed;
+ else
+ _remoteFeed = feed;
+ _ = UpdateGrabbersAsync(_scriptsUsed);
+ }
+
+ private async Task UpdateGrabbersAsync(IEnumerable ids)
+ {
+ if (_remoteFeed == null)
+ await UpdateFeedAsync().ConfigureAwait(false);
+
+ var localFeed = _localFeed;
+ var remoteFeed = _remoteFeed;
+ if (localFeed == null || remoteFeed == null)
+ return false;
+
+ var idSet = new HashSet(ids, StringComparer.InvariantCultureIgnoreCase);
+
+ var localScripts = localFeed.GetScripts()
+ .Where(s => idSet.Contains(s.Id))
+ .ToDictionary(s => s.Id);
+ var remoteScripts = remoteFeed.GetScripts()
+ .Where(s => idSet.Contains(s.Id));
+
+ // compare scripts
+ var updateTasks = new List>();
+ foreach (var remoteScript in remoteScripts)
+ {
+ var localScript = localScripts[remoteScript.Id];
+ if (localScript != null && remoteScript.GetVersion() <= localScript.GetVersion())
+ continue;
+ var task = UpdateGrabberAsync(remoteScript.Id);
+ updateTasks.Add(task);
+ }
+ await Task.WhenAll(updateTasks).ConfigureAwait(false);
+ var anyUpdates = updateTasks.Any(t => t.Result);
+
+ if (anyUpdates)
+ {
+ await LoadLocalGrabbers().ConfigureAwait(false);
+ }
+ return anyUpdates;
+ }
+
+ private async Task UpdateGrabberAsync(string id)
+ {
+ // get current records
+ var localScript = _localFeed?.GetScript(id);
+ var remoteScript = _remoteFeed?.GetScript(id);
+ if (remoteScript == null)
+ return false;
+ if (localScript != null && localScript.GetVersion() >= remoteScript.GetVersion())
+ return false;
+
+ if (!_scriptsUpdating.Add(id))
+ return false;
+
+ try
+ {
+ // update script
+ await GetScriptAsync(id);
+ }
+ finally
+ {
+ _scriptsUpdating.Remove(id);
+ }
+ return true;
+ }
+
+ private sealed class BlackWidowGrabber : GrabberBase, IBlackWidowGrabber
+ {
+ private readonly BlackWidowService _service;
+
+ public BlackWidowGrabber(BlackWidowService service, IGrabberServices services) : base(services)
+ {
+ _service = service;
+ }
+
+ public override string StringId => "BlackWidow";
+
+ public override string Name => "BlackWidow";
+
+ public override GrabOptions DefaultGrabOptions { get; } = new GrabOptions(GrabOptionFlags.All);
+
+ public IEnumerable GetScriptGrabbers()
+ {
+ return _service._grabbers.Values.AsEnumerable();
+ }
+
+ public override bool Supports(Uri uri)
+ {
+ return new[] { _service._localFeed, _service._remoteFeed }
+ .Any(feed => feed?.GetScripts().Any(s => s.IsMatch(uri)) ?? false);
+ }
+
+ protected override async Task InternalGrabAsync(Uri uri, CancellationToken cancellationToken, GrabOptions options, IProgress progress)
+ {
+ Dictionary GetGrabbers()
+ => _service._grabbers
+ .Where(g => g.Value.Supports(uri))
+ .ToDictionary(g => g.Key, g => g.Value);
+
+ var grabbers = GetGrabbers();
+ if (await _service.UpdateGrabbersAsync(grabbers.Keys).ConfigureAwait(false))
+ {
+ grabbers = GetGrabbers();
+ }
+
+ foreach (var grabber in grabbers)
+ {
+ _service._scriptsUsed.Add(grabber.Key);
+ var result = await grabber.Value.GrabAsync(uri, cancellationToken, options, progress).ConfigureAwait(false);
+ if (result != null)
+ return result;
+ }
+ return null;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/SharpGrabber.BlackWidow/Builder/BlackWidowBuilder.cs b/src/SharpGrabber.BlackWidow/Builder/BlackWidowBuilder.cs
new file mode 100644
index 0000000..3df3f2c
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/BlackWidowBuilder.cs
@@ -0,0 +1,103 @@
+using System;
+using System.Threading.Tasks;
+using DotNetTools.SharpGrabber.BlackWidow.Host;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter;
+using DotNetTools.SharpGrabber.BlackWidow.Repository;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Build a BlackWidow service.
+ ///
+ public sealed class BlackWidowBuilder : IBlackWidowBuilder
+ {
+ private IGrabberRepository _localRepository;
+ private IGrabberRepository _remoteRepository;
+ private IGrabberServices _grabberServices;
+ private IGrabberScriptInterpreterService _interpreterService;
+ private IScriptHost _scriptHost;
+ private IGrabberRepositoryChangeDetector _changeDetector;
+
+ private BlackWidowBuilder() { }
+
+ ///
+ /// Creates a new .
+ ///
+ public static BlackWidowBuilder New()
+ => new();
+
+ public async Task BuildAsync()
+ {
+ if (_localRepository == null)
+ throw new InvalidOperationException("Local repository is unspecified.");
+ if (_remoteRepository == null)
+ throw new InvalidOperationException("Remote repository is unspecified.");
+ var changeDetector = _changeDetector ?? new GrabberRepositoryChangeDetector(new[] { _localRepository, _remoteRepository });
+ if (_interpreterService == null)
+ SetDefaultInterpreterService();
+ var grabberServices = _grabberServices ?? GrabberServices.Default;
+
+ var service = await BlackWidowService.CreateAsync(_localRepository, _remoteRepository,
+ grabberServices ?? throw new InvalidOperationException("Grabber services instance is unspecified."),
+ _scriptHost ?? new ScriptHost(),
+ _interpreterService, changeDetector).ConfigureAwait(false);
+ return service;
+ }
+
+ public IBlackWidowBuilder ConfigureLocalRepository(Action configurator)
+ {
+ var cfg = new BlackWidowRepositoryConfigurator();
+ configurator(cfg);
+ _localRepository = cfg.Repository ?? throw new InvalidOperationException("No");
+ return this;
+ }
+
+ public IBlackWidowBuilder ConfigureRemoteRepository(Action configurator)
+ {
+ var cfg = new BlackWidowRepositoryConfigurator();
+ configurator(cfg);
+ _remoteRepository = cfg.Repository ?? throw new InvalidOperationException("No");
+ return this;
+ }
+
+ public IBlackWidowBuilder SetChangeDetector(IGrabberRepositoryChangeDetector changeDetector)
+ {
+ _changeDetector = changeDetector;
+ return this;
+ }
+
+ public IBlackWidowBuilder SetScriptHost(IScriptHost scriptHost)
+ {
+ _scriptHost = scriptHost;
+ return this;
+ }
+
+ public IBlackWidowBuilder UseInterpreterService(IGrabberScriptInterpreterService interpreterService)
+ {
+ _interpreterService = interpreterService;
+ return this;
+ }
+
+ public IBlackWidowBuilder SetGrabberServices(IGrabberServices grabberServices)
+ {
+ _grabberServices = grabberServices;
+ return this;
+ }
+
+ public IBlackWidowBuilder ConfigureInterpreterService(Action configure)
+ {
+ var configurator = new GrabberScriptInterpreterServiceConfigurator()
+ .UseScriptHost(_scriptHost);
+ configure(configurator);
+
+ var interpreterService = configurator.Build();
+ return UseInterpreterService(interpreterService);
+ }
+
+ private void SetDefaultInterpreterService()
+ {
+ var service = new GrabberScriptInterpreterService();
+ ConfigureInterpreterService(cfg => cfg.AddJint());
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/BlackWidowRepositoryConfigurator.cs b/src/SharpGrabber.BlackWidow/Builder/BlackWidowRepositoryConfigurator.cs
new file mode 100644
index 0000000..17a7cda
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/BlackWidowRepositoryConfigurator.cs
@@ -0,0 +1,19 @@
+using DotNetTools.SharpGrabber.BlackWidow.Repository;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Builds BlackWidow repositories.
+ ///
+ internal class BlackWidowRepositoryConfigurator : IBlackWidowRepositoryConfigurator
+ {
+ public IGrabberRepository Repository { get; private set; }
+
+ public IBlackWidowRepositoryConfigurator Use(IGrabberRepository repository)
+ {
+ Repository?.Dispose();
+ Repository = repository;
+ return this;
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/BuilderExtensions.cs b/src/SharpGrabber.BlackWidow/Builder/BuilderExtensions.cs
new file mode 100644
index 0000000..70d66e1
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/BuilderExtensions.cs
@@ -0,0 +1,31 @@
+using DotNetTools.SharpGrabber.BlackWidow.Definitions;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter.JavaScript;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Defines extension methods for builder and configurator interfaces to work with built-in implementations.
+ ///
+ public static class BuilderExtensions
+ {
+ ///
+ /// Registers Jint as the JavaScript interpreter.
+ ///
+ public static IGrabberScriptInterpreterServiceConfigurator AddJint(this IGrabberScriptInterpreterServiceConfigurator interpreterService)
+ {
+ return interpreterService.AddInterpreter(GrabberScriptType.JavaScript, context =>
+ {
+ return new JintJavaScriptInterpreter(context.ApiService, context.GrabberServices, context.ScriptHost);
+ });
+ }
+
+ ///
+ /// Configures to use the official API service.
+ ///
+ public static IGrabberScriptInterpreterServiceConfigurator SetDefaultApiService(this IGrabberScriptInterpreterServiceConfigurator interpreterService)
+ {
+ return interpreterService.SetApiService(context => new DefaultInterpreterApiService(context.GrabberServices, context.GrabbedTypes, context.TypeConverter));
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterActivationContext.cs b/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterActivationContext.cs
new file mode 100644
index 0000000..5551b15
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterActivationContext.cs
@@ -0,0 +1,34 @@
+using System;
+using DotNetTools.SharpGrabber.BlackWidow.Host;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Provides references to services used when activating an interpreter.
+ ///
+ public class GrabberScriptInterpreterActivationContext
+ {
+ public GrabberScriptInterpreterActivationContext(IInterpreterApiService apiService, IGrabberServices grabberServices, IScriptHost scripHost)
+ {
+ ApiService = apiService ?? throw new ArgumentNullException(nameof(apiService));
+ GrabberServices = grabberServices ?? throw new ArgumentNullException(nameof(grabberServices));
+ ScriptHost = scripHost ?? throw new ArgumentNullException(nameof(scripHost));
+ }
+
+ ///
+ /// Gets the interpreter API service.
+ ///
+ public IInterpreterApiService ApiService { get; }
+
+ ///
+ /// Gets the grabber services.
+ ///
+ public IGrabberServices GrabberServices { get; }
+
+ ///
+ /// Gets the script host.
+ ///
+ public IScriptHost ScriptHost { get; }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterApiServiceActivationContext.cs b/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterApiServiceActivationContext.cs
new file mode 100644
index 0000000..aa2956e
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterApiServiceActivationContext.cs
@@ -0,0 +1,41 @@
+using System;
+using DotNetTools.ConvertEx;
+using DotNetTools.SharpGrabber.BlackWidow.Host;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Provides references to services used when activating an interpreter API service.
+ ///
+ public class GrabberScriptInterpreterApiServiceActivationContext
+ {
+ public GrabberScriptInterpreterApiServiceActivationContext(IGrabberServices grabberServices, IScriptHost scriptHost,
+ IGrabbedTypeCollection grabbedTypes, ITypeConverter typeConverter)
+ {
+ GrabberServices = grabberServices ?? throw new ArgumentNullException(nameof(grabberServices));
+ ScriptHost = scriptHost ?? throw new ArgumentNullException(nameof(scriptHost));
+ GrabbedTypes = grabbedTypes ?? throw new ArgumentNullException(nameof(grabbedTypes));
+ TypeConverter = typeConverter ?? throw new ArgumentNullException(nameof(typeConverter));
+ }
+
+ ///
+ /// Gets the grabber services.
+ ///
+ public IGrabberServices GrabberServices { get; }
+
+ ///
+ /// Gets the script host.
+ ///
+ public IScriptHost ScriptHost { get; }
+
+ ///
+ /// Gets the collection of grabbed types.
+ ///
+ public IGrabbedTypeCollection GrabbedTypes { get; }
+
+ ///
+ /// Gets the type converter.
+ ///
+ public ITypeConverter TypeConverter { get; }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterServiceConfigurator.cs b/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterServiceConfigurator.cs
new file mode 100644
index 0000000..4585ef1
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/GrabberScriptInterpreterServiceConfigurator.cs
@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using DotNetTools.ConvertEx;
+using DotNetTools.SharpGrabber.BlackWidow.Definitions;
+using DotNetTools.SharpGrabber.BlackWidow.Host;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api;
+using DotNetTools.SharpGrabber.BlackWidow.TypeConversion;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Builds .
+ ///
+ internal class GrabberScriptInterpreterServiceConfigurator : IGrabberScriptInterpreterServiceConfigurator
+ {
+ private readonly Dictionary> _interpreterFactories = new();
+ private Func _apiServiceFactory;
+ private IGrabberServices _grabberServices;
+ private IScriptHost _scriptHost;
+ private IGrabbedTypeCollection _grabbedTypeCollection;
+ private ITypeConverter _typeConverter;
+
+ public IGrabberScriptInterpreterServiceConfigurator UseGrabberServices(IGrabberServices grabberServices)
+ {
+ _grabberServices = grabberServices;
+ return this;
+ }
+
+ public IGrabberScriptInterpreterServiceConfigurator UseScriptHost(IScriptHost scriptHost)
+ {
+ _scriptHost = scriptHost;
+ return this;
+ }
+
+ public IGrabberScriptInterpreterServiceConfigurator UseGrabbedTypeCollection(IGrabbedTypeCollection grabbedTypeCollection)
+ {
+ _grabbedTypeCollection = grabbedTypeCollection;
+ return this;
+ }
+
+ public IGrabberScriptInterpreterServiceConfigurator UseTypeConverter(ITypeConverter typeConverter)
+ {
+ _typeConverter = typeConverter;
+ return this;
+ }
+
+ public IGrabberScriptInterpreterServiceConfigurator SetApiService(Func apiServiceFactory)
+ {
+ _apiServiceFactory = apiServiceFactory;
+ return this;
+ }
+
+ public IGrabberScriptInterpreterServiceConfigurator AddInterpreter(GrabberScriptType scriptType,
+ Func interpreterFactory)
+ {
+ _interpreterFactories[scriptType] = interpreterFactory;
+ return this;
+ }
+
+ public IGrabberScriptInterpreterService Build()
+ {
+ if (_apiServiceFactory == null)
+ this.SetDefaultApiService();
+ if (_apiServiceFactory == null)
+ throw new InvalidOperationException("Interpreter API service is unspecified.");
+
+ var grabberServies = _grabberServices ?? GrabberServices.Default;
+ var scriptHost = _scriptHost ?? new ScriptHost();
+ var grabbedTypeCollection = _grabbedTypeCollection ?? new GrabbedTypeCollection();
+ var typeConverter = _typeConverter ?? TypeConverters.Default;
+ var apiServiceContext = new GrabberScriptInterpreterApiServiceActivationContext(grabberServies, scriptHost, grabbedTypeCollection, typeConverter);
+ var apiService = _apiServiceFactory.Invoke(apiServiceContext);
+ var interpreterContext = new GrabberScriptInterpreterActivationContext(apiService, grabberServies, scriptHost);
+
+ var service = new GrabberScriptInterpreterService();
+ foreach (var pair in _interpreterFactories)
+ {
+ var interpreter = pair.Value(interpreterContext);
+ service.Register(pair.Key, interpreter);
+ }
+ return service;
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/IBlackWidowBuilder.cs b/src/SharpGrabber.BlackWidow/Builder/IBlackWidowBuilder.cs
new file mode 100644
index 0000000..e8dbb4e
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/IBlackWidowBuilder.cs
@@ -0,0 +1,55 @@
+using System;
+using System.Threading.Tasks;
+using DotNetTools.SharpGrabber.BlackWidow.Host;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter;
+using DotNetTools.SharpGrabber.BlackWidow.Repository;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Builds a .
+ ///
+ public interface IBlackWidowBuilder
+ {
+ ///
+ /// Configures the local repository.
+ ///
+ IBlackWidowBuilder ConfigureLocalRepository(Action configure);
+
+ ///
+ /// Configures the remote repository.
+ ///
+ IBlackWidowBuilder ConfigureRemoteRepository(Action configure);
+
+ ///
+ /// Sets the grabber services.
+ ///
+ IBlackWidowBuilder SetGrabberServices(IGrabberServices grabberServices);
+
+ ///
+ /// Sets the script host.
+ ///
+ IBlackWidowBuilder SetScriptHost(IScriptHost scriptHost);
+
+ ///
+ /// Sets the change detector.
+ ///
+ IBlackWidowBuilder SetChangeDetector(IGrabberRepositoryChangeDetector changeDetector);
+
+ ///
+ /// Sets to be used.
+ ///
+ IBlackWidowBuilder UseInterpreterService(IGrabberScriptInterpreterService interpreterService);
+
+ ///
+ /// Configures the interpreter service.
+ ///
+ IBlackWidowBuilder ConfigureInterpreterService(Action configure);
+
+ ///
+ /// Builds the service.
+ ///
+ /// Thrown in case of misconfiguration.
+ Task BuildAsync();
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/IBlackWidowRepositoryConfigurator.cs b/src/SharpGrabber.BlackWidow/Builder/IBlackWidowRepositoryConfigurator.cs
new file mode 100644
index 0000000..d030e38
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/IBlackWidowRepositoryConfigurator.cs
@@ -0,0 +1,20 @@
+using DotNetTools.SharpGrabber.BlackWidow.Repository;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Configures a repository on a builder.
+ ///
+ public interface IBlackWidowRepositoryConfigurator
+ {
+ ///
+ /// Gets the configured repository.
+ ///
+ IGrabberRepository Repository { get; }
+
+ ///
+ /// Uses a repository instance.
+ ///
+ IBlackWidowRepositoryConfigurator Use(IGrabberRepository repository);
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Builder/IGrabberScriptInterpreterServiceConfigurator.cs b/src/SharpGrabber.BlackWidow/Builder/IGrabberScriptInterpreterServiceConfigurator.cs
new file mode 100644
index 0000000..0d507b5
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Builder/IGrabberScriptInterpreterServiceConfigurator.cs
@@ -0,0 +1,52 @@
+using System;
+using DotNetTools.ConvertEx;
+using DotNetTools.SharpGrabber.BlackWidow.Definitions;
+using DotNetTools.SharpGrabber.BlackWidow.Host;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter;
+using DotNetTools.SharpGrabber.BlackWidow.Interpreter.Api;
+
+namespace DotNetTools.SharpGrabber.BlackWidow
+{
+ ///
+ /// Configures a for a BlackWidow builder.
+ ///
+ public interface IGrabberScriptInterpreterServiceConfigurator
+ {
+ ///
+ /// Configures the builder to use .
+ ///
+ IGrabberScriptInterpreterServiceConfigurator UseGrabberServices(IGrabberServices grabberServices);
+
+ ///
+ /// Configures the builder to use .
+ ///
+ IGrabberScriptInterpreterServiceConfigurator UseScriptHost(IScriptHost scriptHost);
+
+ ///
+ /// Configures the builder to use .
+ ///
+ IGrabberScriptInterpreterServiceConfigurator UseGrabbedTypeCollection(IGrabbedTypeCollection grabbedTypeCollection);
+
+ ///
+ /// Configures the builder to use .
+ ///
+ IGrabberScriptInterpreterServiceConfigurator UseTypeConverter(ITypeConverter typeConverter);
+
+ ///
+ /// Sets an interpreter API service factory.
+ ///
+ IGrabberScriptInterpreterServiceConfigurator SetApiService(Func apiServiceFactory);
+
+ ///
+ /// Registers an interpreter factory.
+ ///
+ IGrabberScriptInterpreterServiceConfigurator AddInterpreter(GrabberScriptType scriptType,
+ Func interpreterFactory);
+
+ ///
+ /// Builds a configured instance of .
+ ///
+ /// Thrown in case of missing information.
+ IGrabberScriptInterpreterService Build();
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptSource.cs b/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptSource.cs
new file mode 100644
index 0000000..5312d29
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptSource.cs
@@ -0,0 +1,38 @@
+using System.IO;
+using System.Threading.Tasks;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Definitions
+{
+ ///
+ /// Default implementation for
+ ///
+ public class GrabberScriptSource : IGrabberScriptSource
+ {
+ ///
+ /// Refers to a static empty source.
+ ///
+ public static readonly GrabberScriptSource Empty = new(string.Empty);
+
+ private readonly string _source;
+
+ public GrabberScriptSource(string source)
+ {
+ _source = source;
+ }
+
+ ///
+ /// Creates a by reading all the source code from a file.
+ ///
+ public static GrabberScriptSource FromFile(string fileName)
+ {
+ var src = File.ReadAllText(fileName);
+ return new GrabberScriptSource(src);
+ }
+
+ public string GetSource()
+ => _source;
+
+ public Task GetSourceAsync()
+ => Task.FromResult(_source);
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptType.cs b/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptType.cs
new file mode 100644
index 0000000..3f2e4e4
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptType.cs
@@ -0,0 +1,33 @@
+using System.Linq;
+using System.Reflection;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Definitions
+{
+ ///
+ /// Defines all possible script types.
+ ///
+ public enum GrabberScriptType
+ {
+ ///
+ /// ECMAScript
+ ///
+ [GrabberScriptType(FileExtension = "js")]
+ JavaScript = 1,
+ }
+
+ public static class GrabberScriptTypeExtensions
+ {
+ ///
+ /// Gets the associated with the value.
+ ///
+ public static GrabberScriptTypeAttribute GetScriptTypeAttribute(this GrabberScriptType value, bool orDefault = true)
+ {
+ GrabberScriptTypeAttribute GetDefault()
+ => orDefault ? GrabberScriptTypeAttribute.Default : null;
+
+ var enumType = typeof(GrabberScriptType);
+ var member = enumType.GetMember(value.ToString()).FirstOrDefault(m => m.DeclaringType == enumType);
+ return member.GetCustomAttribute() ?? GetDefault();
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptTypeAttribute.cs b/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptTypeAttribute.cs
new file mode 100644
index 0000000..b71c7a9
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Definitions/GrabberScriptTypeAttribute.cs
@@ -0,0 +1,18 @@
+using System;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Definitions
+{
+ [AttributeUsage(AttributeTargets.Field)]
+ public class GrabberScriptTypeAttribute : Attribute
+ {
+ ///
+ /// Gets the default value.
+ ///
+ public static GrabberScriptTypeAttribute Default => new();
+
+ ///
+ /// Gets or sets the file extension associated with this script type.
+ ///
+ public string FileExtension { get; set; }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Definitions/IGrabberScriptSource.cs b/src/SharpGrabber.BlackWidow/Definitions/IGrabberScriptSource.cs
new file mode 100644
index 0000000..033f676
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Definitions/IGrabberScriptSource.cs
@@ -0,0 +1,20 @@
+using System.Threading.Tasks;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Definitions
+{
+ ///
+ /// Provides access to the source of a grabber script.
+ ///
+ public interface IGrabberScriptSource
+ {
+ ///
+ /// Gets the source code of the grabber script.
+ ///
+ string GetSource();
+
+ ///
+ /// Gets the source code of the grabber script.
+ ///
+ Task GetSourceAsync();
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Exceptions/BlackWidowException.cs b/src/SharpGrabber.BlackWidow/Exceptions/BlackWidowException.cs
new file mode 100644
index 0000000..e9f46c6
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Exceptions/BlackWidowException.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Exceptions
+{
+ public class BlackWidowException : Exception
+ {
+ public BlackWidowException()
+ {
+ }
+
+ public BlackWidowException(string message) : base(message)
+ {
+ }
+
+ public BlackWidowException(string message, Exception innerException) : base(message, innerException)
+ {
+ }
+
+ protected BlackWidowException(SerializationInfo info, StreamingContext context) : base(info, context)
+ {
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Exceptions/ScriptApiVersionMismatchException.cs b/src/SharpGrabber.BlackWidow/Exceptions/ScriptApiVersionMismatchException.cs
new file mode 100644
index 0000000..a113468
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Exceptions/ScriptApiVersionMismatchException.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Exceptions
+{
+ public class ScriptApiVersionMismatchException : BlackWidowException
+ {
+ public ScriptApiVersionMismatchException()
+ {
+ }
+
+ public ScriptApiVersionMismatchException(string message) : base(message)
+ {
+ }
+
+ public ScriptApiVersionMismatchException(string message, Exception innerException) : base(message, innerException)
+ {
+ }
+
+ protected ScriptApiVersionMismatchException(SerializationInfo info, StreamingContext context) : base(info, context)
+ {
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Exceptions/ScriptInterpretException.cs b/src/SharpGrabber.BlackWidow/Exceptions/ScriptInterpretException.cs
new file mode 100644
index 0000000..e5f74f6
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Exceptions/ScriptInterpretException.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Exceptions
+{
+ public class ScriptInterpretException : BlackWidowException
+ {
+ public ScriptInterpretException() : this("Script interpret error.")
+ {
+ }
+
+ public ScriptInterpretException(string message) : base(message)
+ {
+ }
+
+ public ScriptInterpretException(string message, Exception innerException) : base(message, innerException)
+ {
+ }
+
+ protected ScriptInterpretException(SerializationInfo info, StreamingContext context) : base(info, context)
+ {
+ }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Host/ConsoleLog.cs b/src/SharpGrabber.BlackWidow/Host/ConsoleLog.cs
new file mode 100644
index 0000000..53ae0c1
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Host/ConsoleLog.cs
@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Host
+{
+ ///
+ /// Describes a log entry.
+ ///
+ public class ConsoleLog
+ {
+ public ConsoleLog(ConsoleLogLevel level, params object[] objects)
+ {
+ Level = level;
+ Objects = objects;
+ }
+
+ ///
+ /// Gets the level.
+ ///
+ public ConsoleLogLevel Level { get; }
+
+ ///
+ /// Gets the logged objects.
+ ///
+ public object[] Objects { get; }
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Host/ConsoleLogLevel.cs b/src/SharpGrabber.BlackWidow/Host/ConsoleLogLevel.cs
new file mode 100644
index 0000000..3a1a478
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Host/ConsoleLogLevel.cs
@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Host
+{
+ public enum ConsoleLogLevel
+ {
+ Log,
+ Debug,
+ Error,
+ Info,
+ Warning,
+ Trace
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Host/IScriptHost.cs b/src/SharpGrabber.BlackWidow/Host/IScriptHost.cs
new file mode 100644
index 0000000..a695a35
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Host/IScriptHost.cs
@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Host
+{
+ ///
+ /// Defines handlers for various operations on the script host.
+ ///
+ public interface IScriptHost
+ {
+ void Alert(object input);
+
+ void Log(ConsoleLog log);
+ }
+}
diff --git a/src/SharpGrabber.BlackWidow/Host/ScriptHost.cs b/src/SharpGrabber.BlackWidow/Host/ScriptHost.cs
new file mode 100644
index 0000000..ccb6403
--- /dev/null
+++ b/src/SharpGrabber.BlackWidow/Host/ScriptHost.cs
@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DotNetTools.SharpGrabber.BlackWidow.Host
+{
+ ///
+ /// Implements with events.
+ ///
+ public class ScriptHost : IScriptHost
+ {
+ public ScriptHost()
+ {
+ }
+
+ public event Action