AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
"""Reverse URL matching, and the guarded page fetch built for following links."""
import socket
import aiohttp
import pytest
from aiohttp import web
from aiohttp.test_utils import TestServer
from aliens_eye import api
from aliens_eye.core import netguard
from aliens_eye.core.expand import _username_from_url
from aliens_eye.core.urlmatch import ProfileUrlMatcher
SITES = {
"github": "https://github.com/{}",
"github.sponsors": "https://github.com/sponsors/{}",
"reddit": "https://www.reddit.com/user/{}",
"medium": "https://medium.com/@{}",
"tumblr": "https://{}.tumblr.com",
"forum": "https://forum.example/member.php?mode=view&username={}",
"lor": "https://www.linux.example/people/{}/profile",
"twitter": "https://twitter.com/{}",
"boards": "https://www.pinterest.com/{}/{}",
}
@pytest.fixture
def matcher():
return ProfileUrlMatcher(SITES)
def first(matcher, url):
matches = matcher.match(url)
return tuple(matches[0]) if matches else None
@pytest.mark.parametrize("url,expected", [
("https://github.com/arx", ("github", "arx")),
("http://www.GitHub.com/arx/", ("github", "arx")), # scheme, www, case, slash
("https://github.com/arx?tab=repositories#x", ("github", "arx")), # query and fragment ignored
("https://www.reddit.com/user/spez", ("reddit", "spez")),
("https://old.reddit.com/user/spez", ("reddit", "spez")), # host alias
("https://medium.com/@someone", ("medium", "someone")),
("https://arx.tumblr.com", ("tumblr", "arx")),
("https://arx.tumblr.com/post/1", ("tumblr", "arx")),
("https://forum.example/member.php?username=zed&mode=view", ("forum", "zed")),
("https://www.linux.example/people/maxcom/profile", ("lor", "maxcom")),
("https://x.com/arx", ("twitter", "arx")), # x.com is twitter
("https://twitter.com/@arx", ("twitter", "arx")), # leading @ dropped
])
def test_profile_urls_are_recognised(matcher, url, expected):
assert first(matcher, url) == expected
@pytest.mark.parametrize("url", [
"https://github.com/arx/some-repo", # a repository, not a profile
"https://github.com/sponsors", # fixed segment of a sibling template
"https://github.com/features", # site navigation
"https://github.com/login",
"https://github.com/",
"https://twitter.com/intent/tweet?text=hi",
"https://www.tumblr.com", # no username subdomain
"https://www.tumblr.com/explore",
"https://api.tumblr.com", # service subdomain
"https://forum.example/member.php?username=zed", # fixed param missing
"https://forum.example/other.php?mode=view&username=zed", # wrong path
"https://www.linux.example/people/maxcom", # template suffix missing
"https://github.com/logo.png", # a file
"https://github.com/a", # too short to scan
"https://unknown.example/arx",
"javascript:alert(1)",
"file:///etc/passwd",
"not a url",
"",
])
def test_non_profile_urls_do_not_match(matcher, url):
assert matcher.match(url) == []
def test_most_specific_template_wins(matcher):
assert first(matcher, "https://github.com/sponsors/arx") == ("github.sponsors", "arx")
def test_two_placeholder_templates_are_skipped(matcher):
assert all(m.site != "boards" for m in matcher.match("https://www.pinterest.com/arx/board"))
def test_username_case_is_preserved(matcher):
assert first(matcher, "https://github.com/ArxHR") == ("github", "ArxHR")
def test_percent_encoded_usernames_are_decoded(matcher):
assert first(matcher, "https://github.com/%E5%9D%A6%E9%80%94") == ("github", "坦途")
def test_api_matches_against_the_real_catalogue():
assert tuple(api.match_profile_url("https://www.instagram.com/_arx_/")[0]) == ("instagram", "_arx_")
assert tuple(api.match_profile_url("https://www.linkedin.com/in/some-one")[0]) == ("linkedin", "some-one")
assert api.match_profile_url("https://github.com/arx/repo") == []
def test_profile_url_round_trips_through_the_matcher():
url = api.profile_url("github", "arx")
assert tuple(api.match_profile_url(url)[0]) == ("github", "arx")
with pytest.raises(KeyError):
api.profile_url("no-such-site", "arx")
def test_expand_understands_any_catalogue_site():
"""It used to know 19 hard-coded domains and take the first path segment."""
assert _username_from_url("https://www.linkedin.com/in/some-one") == "some-one"
assert _username_from_url("https://twitter.com/charlie") == "charlie"
assert _username_from_url("https://github.com/arx/repo") is None
# --- fetch_page ----------------------------------------------------------------
@pytest.fixture
async def page_server():
hits = {"secret": 0}
app = web.Application()
async def page(request):
return web.Response(text="<html><a rel='me' href='https://github.com/arx'>gh</a></html>",
content_type="text/html")
async def to_internal(request):
raise web.HTTPFound(f"http://127.0.0.1:{request.url.port}/secret")
async def secret(request):
hits["secret"] += 1
return web.Response(text="internal")
app.router.add_get("/page", page)
app.router.add_get("/to-internal", to_internal)
app.router.add_get("/secret", secret)
server = TestServer(app, host="127.0.0.1")
await server.start_server()
server.hits = hits
yield server
await server.close()
async def test_fetch_page_returns_html(page_server):
result = await api.fetch_page(f"http://127.0.0.1:{page_server.port}/page", allow_private=True)
assert result["status"] == 200
assert result["error"] is None
assert "rel='me'" in result["html"]
async def test_fetch_page_refuses_private_addresses_by_default(page_server):
result = await api.fetch_page(f"http://127.0.0.1:{page_server.port}/secret")
assert result["error"]
assert result["html"] == ""
assert page_server.hits["secret"] == 0
async def test_fetch_page_does_not_follow_a_redirect_inward(page_server, monkeypatch):
async def only_localhost(host):
return host == "localhost"
monkeypatch.setattr(netguard, "host_is_public", only_localhost)
# Plain IPv4 connector: the public-only resolver would (rightly) refuse the
# test server itself. The per-hop check is what is under test here.
monkeypatch.setattr(
"aliens_eye.core.scanner.build_session",
lambda config, limit, headers=None, resolver=None: aiohttp.ClientSession(
connector=aiohttp.TCPConnector(family=socket.AF_INET)
),
)
result = await api.fetch_page(f"http://localhost:{page_server.port}/to-internal")
assert result["error"]
assert page_server.hits["secret"] == 0
@pytest.mark.parametrize("url", ["file:///etc/passwd", "javascript:alert(1)", "", "ftp://x/y"])
async def test_fetch_page_never_raises_on_bad_urls(url):
result = await api.fetch_page(url)
assert result["error"] and result["html"] == ""
def test_youtube_is_checked_by_handle():
"""youtube.com/<name> exists only for older channels; every channel has an @handle."""
from aliens_eye import api
assert api.profile_url("youtube", "mkbhd") == "https://www.youtube.com/@mkbhd"
assert api.match_profile_url("https://www.youtube.com/@mkbhd") == [("youtube", "mkbhd")]
assert api.match_profile_url("https://youtube.com/@MKBHD/videos") == [] # a tab, not the profile
assert api.match_profile_url("https://www.youtube.com/c/mkbhd") == [("youtube channel", "mkbhd")]
assert api.match_profile_url("https://www.youtube.com/watch?v=abc") == []