Building 70 Projects ranging from beginner to advanced so anyone can — learn from, build upon, use as a reference, or even copy directly. Owned by my cybersecurity company 👇
Go aicertificationcloud-engineeringcybercybersecuritycybersecurity-analystcybersecurity-certificationcybersecurity-certificationscybersecurity-educationcybersecurity-jobscybersecurity-portfoliocybersecurity-projectscybersecurity-toolsgrchackinglearningpentestingproject-repositoryprojetcspython
{-
©AngelaMos | 2026
Features.hs
-}
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RecordWildCards #-}
module Aenebris.ML.Features
( FeatureVector(..)
, FeatureContext(..)
, emptyFeatureContext
, extractFeatures
, featureVectorLength
, featureVectorToList
, featureVectorToVector
, featureNames
, headerCountCap
, pathDepthCap
, queryParamCountCap
, userAgentLengthCap
, pathEntropyMax
, acceptValueLengthCap
, botKeywordPatterns
, headlessPatterns
, suspiciousPathExtensions
, commonBrowserMarkers
, idempotentMethods
, commonBrowserMarkerThreshold
, secFetchModeValidValues
, secFetchDestNavigationValues
, headerOrderTrackedNames
, chromeHeaderCanonicalOrder
, firefoxHeaderCanonicalOrder
, shannonEntropyBytes
, uaSecChConsistency
, uaContainsBotKeyword
, uaContainsHeadlessMarker
, uaIsCommonBrowser
, uaPlatformConsistency
, pathDepth
, pathHasSuspiciousExtension
, methodIsIdempotent
, secFetchModeIsValid
, secFetchTripleIsCoherent
, acceptIsWildcard
, headerOrderIsCanonicalBrowser
, clamp01
, normalizedRatio
) where
import Aenebris.Geo (GeoInfo(..), emptyGeoInfo)
import Data.ByteString (ByteString)
import qualified Data.ByteString as BS
import qualified Data.ByteString.Char8 as BC
import Data.CaseInsensitive (CI)
import Data.Char (toLower)
import Data.Map.Strict (Map)
import qualified Data.Map.Strict as Map
import Data.Maybe (fromMaybe, isJust)
import Data.Text (Text)
import Data.Vector.Unboxed (Vector)
import qualified Data.Vector.Unboxed as VU
import Data.Word (Word8)
import Network.Wai
( Request
, queryString
, rawPathInfo
, requestHeaders
, requestMethod
)
headerCountCap :: Double
headerCountCap = 32.0
pathDepthCap :: Double
pathDepthCap = 16.0
queryParamCountCap :: Double
queryParamCountCap = 32.0
userAgentLengthCap :: Double
userAgentLengthCap = 256.0
pathEntropyMax :: Double
pathEntropyMax = 8.0
acceptValueLengthCap :: Double
acceptValueLengthCap = 200.0
commonBrowserMarkerThreshold :: Int
commonBrowserMarkerThreshold = 2
secFetchModeValidValues :: [ByteString]
secFetchModeValidValues =
["cors", "no-cors", "navigate", "same-origin", "websocket"]
secFetchDestNavigationValues :: [ByteString]
secFetchDestNavigationValues =
["document", "iframe", "frame", "embed", "object"]
secFetchSiteValidValues :: [ByteString]
secFetchSiteValidValues =
["none", "same-origin", "same-site", "cross-site"]
headerOrderTrackedNames :: [CI ByteString]
headerOrderTrackedNames =
["host", "user-agent", "accept", "accept-encoding", "accept-language"]
chromeHeaderCanonicalOrder :: [CI ByteString]
chromeHeaderCanonicalOrder =
["host", "user-agent", "accept", "accept-encoding", "accept-language"]
firefoxHeaderCanonicalOrder :: [CI ByteString]
firefoxHeaderCanonicalOrder =
["host", "user-agent", "accept", "accept-language", "accept-encoding"]
botKeywordPatterns :: [ByteString]
botKeywordPatterns =
[ "bot"
, "crawl"
, "spider"
, "scrape"
, "fetch"
, "monitor"
, "checker"
, "preview"
, "wget"
, "curl"
, "python-requests"
, "go-http-client"
, "java/"
, "okhttp"
, "libwww"
, "httpclient"
]
headlessPatterns :: [ByteString]
headlessPatterns =
[ "headlesschrome"
, "phantomjs"
, "puppeteer"
, "playwright"
, "selenium"
, "webdriver"
, "nightmare"
, "splash"
]
suspiciousPathExtensions :: [ByteString]
suspiciousPathExtensions =
[ ".php"
, ".asp"
, ".aspx"
, ".cgi"
, ".env"
, ".bak"
, ".sql"
, ".log"
, ".swp"
, ".old"
, ".htaccess"
, ".htpasswd"
, ".git"
, ".ini"
, ".conf"
, ".yml"
, ".yaml"
, ".pem"
, ".key"
]
commonBrowserMarkers :: [ByteString]
commonBrowserMarkers =
[ "mozilla/5.0"
, "applewebkit"
, "chrome/"
, "safari/"
, "firefox/"
, "edg/"
, "gecko/"
, "version/"
]
idempotentMethods :: [ByteString]
idempotentMethods = ["GET", "HEAD", "OPTIONS", "TRACE"]
chromiumFamilyMarkers :: [ByteString]
chromiumFamilyMarkers = ["chrome/", "edg/", "opr/", "chromium/", "yabrowser/"]
acceptLanguageHeader :: CI ByteString
acceptLanguageHeader = "accept-language"
userAgentHeader :: CI ByteString
userAgentHeader = "user-agent"
acceptEncodingHeader :: CI ByteString
acceptEncodingHeader = "accept-encoding"
refererHeader :: CI ByteString
refererHeader = "referer"
cookieHeader :: CI ByteString
cookieHeader = "cookie"
secChUaHeader :: CI ByteString
secChUaHeader = "sec-ch-ua"
secChUaPlatformHeader :: CI ByteString
secChUaPlatformHeader = "sec-ch-ua-platform"
secFetchSiteHeader :: CI ByteString
secFetchSiteHeader = "sec-fetch-site"
secFetchModeHeader :: CI ByteString
secFetchModeHeader = "sec-fetch-mode"
secFetchDestHeader :: CI ByteString
secFetchDestHeader = "sec-fetch-dest"
acceptHeader :: CI ByteString
acceptHeader = "accept"
clamp01 :: Double -> Double
clamp01 x
| x < 0.0 = 0.0
| x > 1.0 = 1.0
| otherwise = x
normalizedRatio :: Int -> Double -> Double
normalizedRatio n cap
| cap <= 0.0 = 0.0
| otherwise = clamp01 (fromIntegral n / cap)
asciiToLower :: ByteString -> ByteString
asciiToLower = BC.map toLower
shannonEntropyBytes :: ByteString -> Double
shannonEntropyBytes bs
| BS.null bs = 0.0
| otherwise =
let !total = fromIntegral (BS.length bs) :: Double
freq :: Map Word8 Int
freq = BS.foldl'
(\m b -> Map.insertWith (+) b 1 m)
Map.empty
bs
step !acc !c =
let p = fromIntegral c / total
in acc - p * logBase 2 p
in Map.foldl' step 0.0 freq
uaSecChConsistency :: Maybe ByteString -> Maybe ByteString -> Double
uaSecChConsistency mUa mSecCh = case mSecCh of
Nothing -> 1.0
Just _ -> case mUa of
Nothing -> 0.0
Just ua ->
let lower = asciiToLower ua
isChromium = any (`BS.isInfixOf` lower) chromiumFamilyMarkers
in if isChromium then 1.0 else 0.0
uaContainsBotKeyword :: ByteString -> Bool
uaContainsBotKeyword ua =
let lower = asciiToLower ua
in any (`BS.isInfixOf` lower) botKeywordPatterns
uaContainsHeadlessMarker :: ByteString -> Bool
uaContainsHeadlessMarker ua =
let lower = asciiToLower ua
in any (`BS.isInfixOf` lower) headlessPatterns
uaIsCommonBrowser :: ByteString -> Bool
uaIsCommonBrowser ua =
let lower = asciiToLower ua
hits = length (filter (`BS.isInfixOf` lower) commonBrowserMarkers)
in hits >= commonBrowserMarkerThreshold
pathDepth :: ByteString -> Int
pathDepth path =
let segments = filter (not . BS.null) (BC.split '/' path)
in length segments
pathHasSuspiciousExtension :: ByteString -> Bool
pathHasSuspiciousExtension path =
let lower = asciiToLower path
in any (`BS.isSuffixOf` lower) suspiciousPathExtensions
methodIsIdempotent :: ByteString -> Bool
methodIsIdempotent m = m `elem` idempotentMethods
secFetchModeIsValid :: ByteString -> Bool
secFetchModeIsValid v = asciiToLower v `elem` secFetchModeValidValues
secFetchTripleIsCoherent
:: Maybe ByteString
-> Maybe ByteString
-> Maybe ByteString
-> Bool
secFetchTripleIsCoherent mSite mMode mDest =
case (asciiToLower <$> mSite, asciiToLower <$> mMode, asciiToLower <$> mDest) of
(Just site, Just mode, Just dest) ->
site `elem` secFetchSiteValidValues
&& mode `elem` secFetchModeValidValues
&& tripleConsistent site mode dest
_ -> False
where
tripleConsistent site mode dest
| site == "none" && mode /= "navigate" = False
| mode == "navigate" && dest `notElem` secFetchDestNavigationValues = False
| mode == "cors" && site == "none" = False
| mode == "websocket" && dest /= "websocket" && dest /= "empty" = False
| otherwise = True
uaPlatformConsistency :: Maybe ByteString -> Maybe ByteString -> Double
uaPlatformConsistency mUa mPlatform = case mPlatform of
Nothing -> 1.0
Just p ->
let lowerP = asciiToLower (stripQuotes p)
in case mUa of
Nothing -> 0.0
Just ua ->
let lowerUa = asciiToLower ua
markers = platformUaMarkers lowerP
in if null markers
then 0.0
else
if any (`BS.isInfixOf` lowerUa) markers
then 1.0
else 0.0
stripQuotes :: ByteString -> ByteString
stripQuotes bs =
let trimmed = BS.dropWhile (== quoteByte) (BS.reverse bs)
back = BS.reverse trimmed
in BS.dropWhile (== quoteByte) back
where
quoteByte = 34
platformUaMarkers :: ByteString -> [ByteString]
platformUaMarkers p
| p == "windows" = ["windows nt"]
| p == "macos" = ["mac os x", "macintosh"]
| p == "linux" = ["linux", "x11"]
| p == "android" = ["android"]
| p == "ios" = ["iphone", "ipad", "ipod"]
| p == "chrome os" = ["cros"]
| p == "chromeos" = ["cros"]
| otherwise = []
acceptIsWildcard :: Maybe ByteString -> Bool
acceptIsWildcard mAccept = case mAccept of
Nothing -> False
Just v ->
let trimmed = BS.dropWhile (== spaceByte)
$ BS.reverse
$ BS.dropWhile (== spaceByte)
$ BS.reverse v
in trimmed == "*/*"
where
spaceByte = 32
headerOrderIsCanonicalBrowser :: [(CI ByteString, ByteString)] -> Bool
headerOrderIsCanonicalBrowser headers =
let projected = filter (`elem` headerOrderTrackedNames)
$ map fst headers
chromeMatch = projected == filter (`elem` projected) chromeHeaderCanonicalOrder
&& all (`elem` projected) requiredCore
firefoxMatch = projected == filter (`elem` projected) firefoxHeaderCanonicalOrder
&& all (`elem` projected) requiredCore
in chromeMatch || firefoxMatch
where
requiredCore = ["host", "user-agent", "accept"]
data FeatureContext = FeatureContext
{ fcGeoInfo :: !GeoInfo
, fcAsnConcentration :: !Double
} deriving (Eq, Show)
emptyFeatureContext :: FeatureContext
emptyFeatureContext = FeatureContext
{ fcGeoInfo = emptyGeoInfo
, fcAsnConcentration = 0.0
}
data FeatureVector = FeatureVector
{ fMissingAcceptLanguage :: {-# UNPACK #-} !Double
, fMissingUserAgent :: {-# UNPACK #-} !Double
, fMissingAcceptEncoding :: {-# UNPACK #-} !Double
, fMissingReferer :: {-# UNPACK #-} !Double
, fHasCookie :: {-# UNPACK #-} !Double
, fHasSecChUa :: {-# UNPACK #-} !Double
, fHeaderCount :: {-# UNPACK #-} !Double
, fUaSecChConsistent :: {-# UNPACK #-} !Double
, fUaBotKeyword :: {-# UNPACK #-} !Double
, fUaHeadless :: {-# UNPACK #-} !Double
, fUaCommonBrowser :: {-# UNPACK #-} !Double
, fUaLength :: {-# UNPACK #-} !Double
, fPathDepth :: {-# UNPACK #-} !Double
, fPathEntropy :: {-# UNPACK #-} !Double
, fSuspiciousPathExt :: {-# UNPACK #-} !Double
, fQueryParamCount :: {-# UNPACK #-} !Double
, fMethodIdempotent :: {-# UNPACK #-} !Double
, fFlaggedAsn :: {-# UNPACK #-} !Double
, fAsnConcentration :: {-# UNPACK #-} !Double
, fCountryUnknown :: {-# UNPACK #-} !Double
, fMissingSecFetchSite :: {-# UNPACK #-} !Double
, fSecFetchModeValid :: {-# UNPACK #-} !Double
, fSecFetchContextCoherent :: {-# UNPACK #-} !Double
, fChUaPlatformPresent :: {-# UNPACK #-} !Double
, fChUaPlatformConsistent :: {-# UNPACK #-} !Double
, fAcceptIsWildcard :: {-# UNPACK #-} !Double
, fAcceptValueLength :: {-# UNPACK #-} !Double
, fHeaderOrderCanonical :: {-# UNPACK #-} !Double
} deriving (Eq, Show)
featureVectorLength :: Int
featureVectorLength = 28
featureNames :: [Text]
featureNames =
[ "missing_accept_language"
, "missing_user_agent"
, "missing_accept_encoding"
, "missing_referer"
, "has_cookie"
, "has_sec_ch_ua"
, "header_count"
, "ua_sec_ch_consistent"
, "ua_bot_keyword"
, "ua_headless"
, "ua_common_browser"
, "ua_length"
, "path_depth"
, "path_entropy"
, "suspicious_path_ext"
, "query_param_count"
, "method_idempotent"
, "flagged_asn"
, "asn_concentration"
, "country_unknown"
, "missing_sec_fetch_site"
, "sec_fetch_mode_valid"
, "sec_fetch_context_coherent"
, "ch_ua_platform_present"
, "ch_ua_platform_consistent"
, "accept_is_wildcard"
, "accept_value_length"
, "header_order_canonical"
]
featureVectorToList :: FeatureVector -> [Double]
featureVectorToList FeatureVector{..} =
[ fMissingAcceptLanguage
, fMissingUserAgent
, fMissingAcceptEncoding
, fMissingReferer
, fHasCookie
, fHasSecChUa
, fHeaderCount
, fUaSecChConsistent
, fUaBotKeyword
, fUaHeadless
, fUaCommonBrowser
, fUaLength
, fPathDepth
, fPathEntropy
, fSuspiciousPathExt
, fQueryParamCount
, fMethodIdempotent
, fFlaggedAsn
, fAsnConcentration
, fCountryUnknown
, fMissingSecFetchSite
, fSecFetchModeValid
, fSecFetchContextCoherent
, fChUaPlatformPresent
, fChUaPlatformConsistent
, fAcceptIsWildcard
, fAcceptValueLength
, fHeaderOrderCanonical
]
featureVectorToVector :: FeatureVector -> Vector Double
featureVectorToVector = VU.fromList . featureVectorToList
boolToDouble :: Bool -> Double
boolToDouble True = 1.0
boolToDouble False = 0.0
extractFeatures :: FeatureContext -> Request -> FeatureVector
extractFeatures FeatureContext{..} req =
let !headers = requestHeaders req
!path = rawPathInfo req
!method = requestMethod req
!mAcceptLang = lookup acceptLanguageHeader headers
!mUserAgent = lookup userAgentHeader headers
!mAcceptEnc = lookup acceptEncodingHeader headers
!mReferer = lookup refererHeader headers
!mCookie = lookup cookieHeader headers
!mSecChUa = lookup secChUaHeader headers
!mSecChPlat = lookup secChUaPlatformHeader headers
!mSecFetchSite = lookup secFetchSiteHeader headers
!mSecFetchMode = lookup secFetchModeHeader headers
!mSecFetchDest = lookup secFetchDestHeader headers
!mAccept = lookup acceptHeader headers
!uaBytes = fromMaybe BS.empty mUserAgent
!uaLen = BS.length uaBytes
!depth = pathDepth path
!entropy = shannonEntropyBytes path
!queryCount = length (queryString req)
!geo = fcGeoInfo
!acceptLen = maybe 0 BS.length mAccept
!modeValid = maybe False secFetchModeIsValid mSecFetchMode
in FeatureVector
{ fMissingAcceptLanguage = boolToDouble (not (isJust mAcceptLang))
, fMissingUserAgent = boolToDouble (not (isJust mUserAgent))
, fMissingAcceptEncoding = boolToDouble (not (isJust mAcceptEnc))
, fMissingReferer = boolToDouble (not (isJust mReferer))
, fHasCookie = boolToDouble (isJust mCookie)
, fHasSecChUa = boolToDouble (isJust mSecChUa)
, fHeaderCount = normalizedRatio (length headers) headerCountCap
, fUaSecChConsistent = uaSecChConsistency mUserAgent mSecChUa
, fUaBotKeyword = boolToDouble
(maybe False uaContainsBotKeyword mUserAgent)
, fUaHeadless = boolToDouble
(maybe False uaContainsHeadlessMarker mUserAgent)
, fUaCommonBrowser = boolToDouble
(maybe False uaIsCommonBrowser mUserAgent)
, fUaLength = normalizedRatio uaLen userAgentLengthCap
, fPathDepth = normalizedRatio depth pathDepthCap
, fPathEntropy = clamp01 (entropy / pathEntropyMax)
, fSuspiciousPathExt = boolToDouble (pathHasSuspiciousExtension path)
, fQueryParamCount = normalizedRatio queryCount queryParamCountCap
, fMethodIdempotent = boolToDouble (methodIsIdempotent method)
, fFlaggedAsn = boolToDouble (giFlaggedAsn geo)
, fAsnConcentration = clamp01 fcAsnConcentration
, fCountryUnknown = boolToDouble
(case giCountryISO geo of
Nothing -> True
Just _ -> False)
, fMissingSecFetchSite = boolToDouble (not (isJust mSecFetchSite))
, fSecFetchModeValid = boolToDouble modeValid
, fSecFetchContextCoherent = boolToDouble
(secFetchTripleIsCoherent
mSecFetchSite mSecFetchMode mSecFetchDest)
, fChUaPlatformPresent = boolToDouble (isJust mSecChPlat)
, fChUaPlatformConsistent = uaPlatformConsistency mUserAgent mSecChPlat
, fAcceptIsWildcard = boolToDouble (acceptIsWildcard mAccept)
, fAcceptValueLength = normalizedRatio acceptLen acceptValueLengthCap
, fHeaderOrderCanonical = boolToDouble
(headerOrderIsCanonicalBrowser headers)
}