More Related Content Similar to A Journey through New Languages - Locaweb Tech Day (20) More from Fabio Akita (20) A Journey through New Languages - Locaweb Tech Day16. #!/usr/bin/env ruby
$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
require 'optparse'
options = { test: false }
option_parser = OptionParser.new do |opts|
opts.banner = "Usage: manga-downloadr [options]"
opts.on("-t", "--test", "Test routine") do |t|
options[:url] = "http://www.mangareader.net/onepunch-man"
options[:name] = "one-punch-man"
options[:directory] = "/tmp/manga-downloadr/one-punch-man"
options[:test] = true
end
opts.on("-u URL", "--url URL",
"Full MangaReader.net manga homepage URL - required") do |v|
options[:url] = v
end
opts.on("-n NAME", "--name NAME",
"slug to be used for the sub-folder to store all manga files - required") do |n|
options[:name] = n
end
opts.on("-d DIRECTORY", "--directory DIRECTORY",
"main folder where all mangas will be stored - required") do |d|
options[:directory] = d
end
opts.on("-h", "--help", "Show this message") do
puts opts
exit
end
end
43. .
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
mix.exs
44. mix.exsdefmodule ExMangaDownloadr.Mixfile do
use Mix.Project
def project do
[app: :ex_manga_downloadr,
version: "1.0.1",
elixir: "~> 1.1",
build_embedded: Mix.env == :prod,
start_permanent: Mix.env == :prod,
escript: [main_module: ExMangaDownloadr.CLI],
deps: deps]
end
# Configuration for the OTP application
#
# Type "mix help compile.app" for more information
def application do
[applications: [:logger, :httpotion, :porcelain],
mod: {PoolManagement, []}]
end
defp deps do
[
{:ibrowse, "~> 4.2.2"},
{:httpotion, "~> 3.0.0"},
{:floki, "~> 0.9.0"},
{:porcelain, "~> 2.0.1"},
{:poolboy, "~> 1.5.1"},
{:mock, "~> 0.1.3", only: :test}
]
end
end
PoolManagement
45. PoolManagement
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
pool_management.ex
46. .
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
pool_management.ex
48. Supervisor
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
supervisor.ex
49. .
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
supervisor.ex
50. supervisor.exdefmodule PoolManagement.Supervisor do
use Supervisor
def start_link do
Supervisor.start_link(__MODULE__, [])
end
def init([]) do
pool_size = System.get_env("POOL_SIZE") || "50"
pool_options = [
name: {:local, :worker_pool},
worker_module: PoolManagement.Worker,
size: String.to_integer(pool_size),
max_overflow: 0
]
children = [
supervisor(Task.Supervisor, [[name: Fetcher.TaskSupervisor,
strategy: :transient, max_restarts: 10]]),
:poolboy.child_spec(:worker_pool, pool_options, [])
]
supervise(children, strategy: :one_for_one)
end
end
Worker
51. supervisor.exdefmodule PoolManagement.Supervisor do
use Supervisor
def start_link do
Supervisor.start_link(__MODULE__, [])
end
def init([]) do
pool_size = System.get_env("POOL_SIZE") || "50"
pool_options = [
name: {:local, :worker_pool},
worker_module: PoolManagement.Worker,
size: String.to_integer(pool_size),
max_overflow: 0
]
children = [
supervisor(Task.Supervisor, [[name: Fetcher.TaskSupervisor,
strategy: :transient, max_restarts: 10]]),
:poolboy.child_spec(:worker_pool, pool_options, [])
]
supervise(children, strategy: :one_for_one)
end
end
Worker
52. supervisor.exdefmodule PoolManagement.Supervisor do
use Supervisor
def start_link do
Supervisor.start_link(__MODULE__, [])
end
def init([]) do
pool_size = System.get_env("POOL_SIZE") || "50"
pool_options = [
name: {:local, :worker_pool},
worker_module: PoolManagement.Worker,
size: String.to_integer(pool_size),
max_overflow: 0
]
children = [
supervisor(Task.Supervisor, [[name: Fetcher.TaskSupervisor,
strategy: :transient, max_restarts: 10]]),
:poolboy.child_spec(:worker_pool, pool_options, [])
]
supervise(children, strategy: :one_for_one)
end
end
Worker
53. supervisor.exdefmodule PoolManagement.Supervisor do
use Supervisor
def start_link do
Supervisor.start_link(__MODULE__, [])
end
def init([]) do
pool_size = System.get_env("POOL_SIZE") || "50"
pool_options = [
name: {:local, :worker_pool},
worker_module: PoolManagement.Worker,
size: String.to_integer(pool_size),
max_overflow: 0
]
children = [
supervisor(Task.Supervisor, [[name: Fetcher.TaskSupervisor,
strategy: :transient, max_restarts: 10]]),
:poolboy.child_spec(:worker_pool, pool_options, [])
]
supervise(children, strategy: :one_for_one)
end
end
Worker
54. Worker
Supervisor
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
worker.ex
55. Supervisor
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
worker.ex
56. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
57. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
58. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
59. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
60. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
61. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
62. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
63. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
64. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
65. worker.exdefmodule PoolManagement.Worker do
use GenServer
# Public APIs
def index_page(url, source) do
end
def chapter_page([chapter_link, source]) do
end
def page_image([page_link, source]) do
end
def page_download_image(image_data, directory) do
end
# internal GenServer implementation
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
end
def handle_call({:page_image, page_link, source}, _from, state) do
end
def handle_call({:page_download_image, image_data, directory}, _from, state)
end
## Helper functions
defp manga_source(source, module) do
case source do
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
end
end
defp download_image({image_src, image_filename}, directory) do
end
end
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
68. defmodule PoolManagement.Worker do
use GenServer
...
def chapter_page([chapter_link, source]) do
Task.Supervisor.async(Fetcher.TaskSupervisor, fn ->
:poolboy.transaction :worker_pool, fn(server) ->
GenServer.call(server, {:chapter_page, chapter_link, source},
@genserver_call_timeout)
end, @task_async_timeout
end)
end
...
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
links = source
|> manga_source("ChapterPage")
|> apply(:pages, [chapter_link])
{:reply, links, state}
end
...
defp manga_source(source, module) do
case source do
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
end
end
end
ChapterPage
69. defmodule PoolManagement.Worker do
use GenServer
...
def chapter_page([chapter_link, source]) do
Task.Supervisor.async(Fetcher.TaskSupervisor, fn ->
:poolboy.transaction :worker_pool, fn(server) ->
GenServer.call(server, {:chapter_page, chapter_link, source},
@genserver_call_timeout)
end, @task_async_timeout
end)
end
...
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
links = source
|> manga_source("ChapterPage")
|> apply(:pages, [chapter_link])
{:reply, links, state}
end
...
defp manga_source(source, module) do
case source do
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
end
end
end
ChapterPage
70. defmodule PoolManagement.Worker do
use GenServer
...
def chapter_page([chapter_link, source]) do
Task.Supervisor.async(Fetcher.TaskSupervisor, fn ->
:poolboy.transaction :worker_pool, fn(server) ->
GenServer.call(server, {:chapter_page, chapter_link, source},
@genserver_call_timeout)
end, @task_async_timeout
end)
end
...
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
links = source
|> manga_source("ChapterPage")
|> apply(:pages, [chapter_link])
{:reply, links, state}
end
...
defp manga_source(source, module) do
case source do
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
end
end
end
ChapterPage
71. defmodule PoolManagement.Worker do
use GenServer
...
def chapter_page([chapter_link, source]) do
Task.Supervisor.async(Fetcher.TaskSupervisor, fn ->
:poolboy.transaction :worker_pool, fn(server) ->
GenServer.call(server, {:chapter_page, chapter_link, source},
@genserver_call_timeout)
end, @task_async_timeout
end)
end
...
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
links = source
|> manga_source("ChapterPage")
|> apply(:pages, [chapter_link])
{:reply, links, state}
end
...
defp manga_source(source, module) do
case source do
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
end
end
end
ChapterPage
72. defmodule PoolManagement.Worker do
use GenServer
...
def chapter_page([chapter_link, source]) do
Task.Supervisor.async(Fetcher.TaskSupervisor, fn ->
:poolboy.transaction :worker_pool, fn(server) ->
GenServer.call(server, {:chapter_page, chapter_link, source},
@genserver_call_timeout)
end, @task_async_timeout
end)
end
...
def handle_call({:chapter_page, chapter_link, source}, _from, state) do
links = source
|> manga_source("ChapterPage")
|> apply(:pages, [chapter_link])
{:reply, links, state}
end
...
defp manga_source(source, module) do
case source do
"mangareader" -> :"Elixir.ExMangaDownloadr.MangaReader.#{module}"
"mangafox" -> :"Elixir.ExMangaDownloadr.Mangafox.#{module}"
end
end
end
ChapterPage
74. ChapterPage
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
cli.ex
76. Workflow
.
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
workflow.ex
77. workflow.exdefmodule ExMangaDownloadr.Workflow do
alias PoolManagement.Worker
require Logger
def chapters({url, source}) do
end
def pages({chapter_list, source}) do
pages_list = chapter_list
|> Enum.map(&Worker.chapter_page([&1, source]))
|> Enum.map(&Task.await(&1, @await_timeout_ms))
|> Enum.reduce([], fn {:ok, list}, acc -> acc ++ list end)
{pages_list, source}
end
def images_sources({pages_list, source}) do
end
def process_downloads(images_list, directory) do
end
def optimize_images(directory) do
Porcelain.shell("mogrify -resize #{@image_dimensions} #{directory}/*.jpg")
directory
end
def compile_pdfs(directory, manga_name) do
end
end
78. workflow.exdefmodule ExMangaDownloadr.Workflow do
alias PoolManagement.Worker
require Logger
def chapters({url, source}) do
end
def pages({chapter_list, source}) do
pages_list = chapter_list
|> Enum.map(&Worker.chapter_page([&1, source]))
|> Enum.map(&Task.await(&1, @await_timeout_ms))
|> Enum.reduce([], fn {:ok, list}, acc -> acc ++ list end)
{pages_list, source}
end
def images_sources({pages_list, source}) do
end
def process_downloads(images_list, directory) do
end
def optimize_images(directory) do
Porcelain.shell("mogrify -resize #{@image_dimensions} #{directory}/*.jpg")
directory
end
def compile_pdfs(directory, manga_name) do
end
end
88. .
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
.
!"" cr_manga_downloadr
!"" libs
# !"" ...
!"" LICENSE
!"" README.md
!"" shard.lock
!"" shard.yml
!"" spec
# !"" cr_manga_downloadr
# # !"" chapters_spec.cr
# # !"" concurrency_spec.cr
# # !"" image_downloader_spec.cr
# # !"" page_image_spec.cr
# # $"" pages_spec.cr
# !"" fixtures
# # !"" ...
# $"" spec_helper.cr
$"" src
!"" cr_manga_downloadr
# !"" chapters.cr
# !"" concurrency.cr
# !"" downloadr_client.cr
# !"" image_downloader.cr
# !"" page_image.cr
# !"" pages.cr
# !"" records.cr
# !"" version.cr
# $"" workflow.cr
$"" cr_manga_downloadr.cr
89. .
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
.
!"" cr_manga_downloadr
!"" libs
# !"" ...
!"" LICENSE
!"" README.md
!"" shard.lock
!"" shard.yml
!"" spec
# !"" cr_manga_downloadr
# # !"" chapters_spec.cr
# # !"" concurrency_spec.cr
# # !"" image_downloader_spec.cr
# # !"" page_image_spec.cr
# # $"" pages_spec.cr
# !"" fixtures
# # !"" ...
# $"" spec_helper.cr
$"" src
!"" cr_manga_downloadr
# !"" chapters.cr
# !"" concurrency.cr
# !"" downloadr_client.cr
# !"" image_downloader.cr
# !"" page_image.cr
# !"" pages.cr
# !"" records.cr
# !"" version.cr
# $"" workflow.cr
$"" cr_manga_downloadr.cr
117. .
!"" _build
# $"" ...
!"" config
# $"" config.exs
!"" deps
# !"" ...
!"" ex_manga_downloadr
!"" lib
# !"" ex_manga_downloadr
# # !"" cli.ex
# # !"" mangafox
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" mangareader
# # # !"" chapter_page.ex
# # # !"" index_page.ex
# # # $"" page.ex
# # !"" pool_management
# # # !"" supervisor.ex
# # # $"" worker.ex
# # $"" workflow.ex
# !"" ex_manga_downloadr.ex
# $"" pool_management.ex
!"" mix.exs
!"" mix.lock
!"" README.md
$"" test
!"" ex_manga_downloadr
# !"" mangafox_test.exs
# $"" mangareader_test.exs
!"" ex_manga_downloadr_test.exs
$"" test_helper.exs
61 directories, 281 files
.
!"" cr_manga_downloadr
!"" libs
# !"" ...
!"" LICENSE
!"" README.md
!"" shard.lock
!"" shard.yml
!"" spec
# !"" cr_manga_downloadr
# # !"" chapters_spec.cr
# # !"" concurrency_spec.cr
# # !"" image_downloader_spec.cr
# # !"" page_image_spec.cr
# # $"" pages_spec.cr
# !"" fixtures
# # !"" ...
# $"" spec_helper.cr
$"" src
!"" cr_manga_downloadr
# !"" chapters.cr
# !"" concurrency.cr
# !"" downloadr_client.cr
# !"" image_downloader.cr
# !"" page_image.cr
# !"" pages.cr
# !"" records.cr
# !"" version.cr
# $"" workflow.cr
$"" cr_manga_downloadr.cr
118. .
!"" cr_manga_downloadr
!"" libs
# !"" ...
!"" LICENSE
!"" README.md
!"" shard.lock
!"" shard.yml
!"" spec
# !"" cr_manga_downloadr
# # !"" chapters_spec.cr
# # !"" concurrency_spec.cr
# # !"" image_downloader_spec.cr
# # !"" page_image_spec.cr
# # $"" pages_spec.cr
# !"" fixtures
# # !"" ...
# $"" spec_helper.cr
$"" src
!"" cr_manga_downloadr
# !"" chapters.cr
# !"" concurrency.cr
# !"" downloadr_client.cr
# !"" image_downloader.cr
# !"" page_image.cr
# !"" pages.cr
# !"" records.cr
# !"" version.cr
# $"" workflow.cr
$"" cr_manga_downloadr.cr
.
!"" bin
# $"" manga-downloadr
!"" Gemfile
!"" Gemfile.lock
!"" lib
# !"" manga-downloadr
# # !"" chapters.rb
# # !"" concurrency.rb
# # !"" downloadr_client.rb
# # !"" image_downloader.rb
# # !"" page_image.rb
# # !"" pages.rb
# # !"" records.rb
# # !"" version.rb
# # $"" workflow.rb
# $"" manga-downloadr.rb
!"" LICENSE.txt
!"" manga-downloadr.gemspec
!"" Rakefile
!"" README.md
$"" spec
!"" fixtures
# !"" ...
!"" manga-downloadr
# !"" chapters_spec.rb
# !"" concurrency_spec.rb
# !"" image_downloader_spec.rb
# !"" page_image_spec.rb
# $"" pages_spec.rb
$"" spec_helper.rb
119. .
!"" cr_manga_downloadr
!"" libs
# !"" ...
!"" LICENSE
!"" README.md
!"" shard.lock
!"" shard.yml
!"" spec
# !"" cr_manga_downloadr
# # !"" chapters_spec.cr
# # !"" concurrency_spec.cr
# # !"" image_downloader_spec.cr
# # !"" page_image_spec.cr
# # $"" pages_spec.cr
# !"" fixtures
# # !"" ...
# $"" spec_helper.cr
$"" src
!"" cr_manga_downloadr
# !"" chapters.cr
# !"" concurrency.cr
# !"" downloadr_client.cr
# !"" image_downloader.cr
# !"" page_image.cr
# !"" pages.cr
# !"" records.cr
# !"" version.cr
# $"" workflow.cr
$"" cr_manga_downloadr.cr
.
!"" bin
# $"" manga-downloadr
!"" Gemfile
!"" Gemfile.lock
!"" lib
# !"" manga-downloadr
# # !"" chapters.rb
# # !"" concurrency.rb
# # !"" downloadr_client.rb
# # !"" image_downloader.rb
# # !"" page_image.rb
# # !"" pages.rb
# # !"" records.rb
# # !"" version.rb
# # $"" workflow.rb
# $"" manga-downloadr.rb
!"" LICENSE.txt
!"" manga-downloadr.gemspec
!"" Rakefile
!"" README.md
$"" spec
!"" fixtures
# !"" ...
!"" manga-downloadr
# !"" chapters_spec.rb
# !"" concurrency_spec.rb
# !"" image_downloader_spec.rb
# !"" page_image_spec.rb
# $"" pages_spec.rb
$"" spec_helper.rb
120. .
!"" cr_manga_downloadr
!"" libs
# !"" ...
!"" LICENSE
!"" README.md
!"" shard.lock
!"" shard.yml
!"" spec
# !"" cr_manga_downloadr
# # !"" chapters_spec.cr
# # !"" concurrency_spec.cr
# # !"" image_downloader_spec.cr
# # !"" page_image_spec.cr
# # $"" pages_spec.cr
# !"" fixtures
# # !"" ...
# $"" spec_helper.cr
$"" src
!"" cr_manga_downloadr
# !"" chapters.cr
# !"" concurrency.cr
# !"" downloadr_client.cr
# !"" image_downloader.cr
# !"" page_image.cr
# !"" pages.cr
# !"" records.cr
# !"" version.cr
# $"" workflow.cr
$"" cr_manga_downloadr.cr
.
!"" bin
# $"" manga-downloadr
!"" Gemfile
!"" Gemfile.lock
!"" lib
# !"" manga-downloadr
# # !"" chapters.rb
# # !"" concurrency.rb
# # !"" downloadr_client.rb
# # !"" image_downloader.rb
# # !"" page_image.rb
# # !"" pages.rb
# # !"" records.rb
# # !"" version.rb
# # $"" workflow.rb
# $"" manga-downloadr.rb
!"" LICENSE.txt
!"" manga-downloadr.gemspec
!"" Rakefile
!"" README.md
$"" spec
!"" fixtures
# !"" ...
!"" manga-downloadr
# !"" chapters_spec.rb
# !"" concurrency_spec.rb
# !"" image_downloader_spec.rb
# !"" page_image_spec.rb
# $"" pages_spec.rb
$"" spec_helper.rb
135. Ruby/Typhoeus
(hydra_concurrency = 30)
38% CPU 1:10 min
Elixir 1.3.4
(POOL_SIZE=60)
148% CPU 1:03 min
Crystal 0.20.1
(opt_batch_size = 60)
18% CPU 1:08 min
Ruby 2.3.1
(opt_batch_size = 50)
21% CPU 1:50 min