diff --git a/datafiles/templates/Html/report.html.st b/datafiles/templates/Html/report.html.st index 0a273bf18..d2c4f02a4 100644 --- a/datafiles/templates/Html/report.html.st +++ b/datafiles/templates/Html/report.html.st @@ -100,16 +100,25 @@ $else$

No log was submitted for this report.

$endif$ -

Test log

+

Test build log

$if(test)$

[view raw]

 $test$
$else$ -

No test log was submitted for this report.

+

No test build log was submitted for this report.

$endif$ +

Test report log

+ +$if(testReport)$ +

[view raw]

+
+$testReport$
+$else$ +

No test report log was submitted for this report.

+$endif$ diff --git a/exes/BuildClient.hs b/exes/BuildClient.hs index 72d337943..fd7b46c97 100644 --- a/exes/BuildClient.hs +++ b/exes/BuildClient.hs @@ -590,9 +590,10 @@ processPkg verbosity opts config docInfo = do let installOk = fmap ("install-outcome: InstallOk" `isInfixOf`) buildReport == Just True -- Run Tests if installOk, Run coverage is Tests runs - (testOutcome, hpcLoc, testfile) <- case installOk && docInfoRunTests docInfo of + (testOutcome, hpcLoc, testfile, testReportFile) <- + case installOk && docInfoRunTests docInfo of True -> testPackage verbosity opts docInfo - False -> return (Nothing, Nothing, Nothing) + False -> return (Nothing, Nothing, Nothing, Nothing) coverageFile <- mapM (coveragePackage verbosity opts docInfo) hpcLoc -- Modify test-outcome and rewrite report file. @@ -601,7 +602,8 @@ processPkg verbosity opts config docInfo = do case bo_dryRun opts of True -> return () False -> uploadResults verbosity config docInfo - mTgz mRpt logfile testfile coverageFile installOk + mTgz mRpt logfile testfile coverageFile + testReportFile installOk where prepareTempBuildDir :: IO () prepareTempBuildDir = do @@ -651,7 +653,7 @@ coveragePackage verbosity opts docInfo loc = do return coverageFile -testPackage :: Verbosity -> BuildOpts -> DocInfo -> IO (Maybe String, Maybe FilePath, Maybe FilePath) +testPackage :: Verbosity -> BuildOpts -> DocInfo -> IO (Maybe String, Maybe FilePath, Maybe FilePath, Maybe FilePath) testPackage verbosity opts docInfo = do let pkgid = docInfoPackage docInfo testLogFile = (installDirectory opts) display pkgid <.> "test" @@ -684,7 +686,7 @@ testPackage verbosity opts docInfo = do [ "Test results for " ++ display pkgid ++ ":" , testResultFile ] - return (testOutcome, hpcLoc, Just testResultFile) + return (testOutcome, hpcLoc, Just testResultFile, Just testReportFile) -- | Build documentation and return @(Just tgz)@ for the built tgz file @@ -882,16 +884,19 @@ tarGzDirectory dir = do where (containing_dir, nested_dir) = splitFileName dir uploadResults :: Verbosity -> BuildConfig -> DocInfo -> Maybe FilePath - -> Maybe FilePath -> FilePath -> Maybe FilePath -> Maybe FilePath -> Bool -> IO () + -> Maybe FilePath -> FilePath -> Maybe FilePath + -> Maybe FilePath -> Maybe FilePath -> Bool -> IO () uploadResults verbosity config docInfo - mdocsTarballFile buildReportFile buildLogFile testLogFile coverageFile installOk = + mdocsTarballFile buildReportFile buildLogFile testLogFile + coverageFile testReportFile installOk = httpSession verbosity "hackage-build" version $ do case mdocsTarballFile of Nothing -> return () Just docsTarballFile -> putDocsTarball config docInfo docsTarballFile - putBuildFiles config docInfo buildReportFile buildLogFile testLogFile coverageFile installOk + putBuildFiles config docInfo buildReportFile buildLogFile testLogFile + coverageFile testReportFile installOk withAuth :: BuildConfig -> Request -> Request withAuth config req = @@ -910,14 +915,24 @@ putDocsTarball config docInfo docsTarballFile = do mEncoding = Just "gzip" putBuildFiles :: BuildConfig -> DocInfo -> Maybe FilePath - -> FilePath -> Maybe FilePath -> Maybe FilePath -> Bool -> HttpSession () -putBuildFiles config docInfo reportFile buildLogFile testLogFile coverageFile installOk = do + -> FilePath -> Maybe FilePath -> Maybe FilePath + -> Maybe FilePath -> Bool -> HttpSession () +putBuildFiles config docInfo reportFile buildLogFile testLogFile coverageFile + testReportFile installOk = do reportContent <- liftIO $ traverse readFile reportFile logContent <- liftIO $ readFile buildLogFile testContent <- liftIO $ traverse readFile testLogFile coverageContent <- liftIO $ traverse readFile coverageFile + testReportCntnt <- + case testReportFile of + Nothing -> pure Nothing + Just fname -> do + exists <- liftIO $ doesFileExist fname + if exists + then Just <$> liftIO (readFile fname) + else pure Nothing let uri = docInfoReports config docInfo - body = encode $ BR.BuildFiles reportContent (Just logContent) testContent coverageContent (not installOk) + body = encode $ BR.BuildFiles reportContent (Just logContent) testContent coverageContent testReportCntnt (not installOk) let headers = [ (hAccept, BSS.pack "application/json") ] req <- withAuth config <$> mkUploadRequest (BSS.pack "PUT") uri "application/json" Nothing headers body runRequest req $ \rsp -> do diff --git a/src/Distribution/Server/Features/BuildReports.hs b/src/Distribution/Server/Features/BuildReports.hs index 7dcd20e09..7a3fb7be8 100644 --- a/src/Distribution/Server/Features/BuildReports.hs +++ b/src/Distribution/Server/Features/BuildReports.hs @@ -6,7 +6,7 @@ module Distribution.Server.Features.BuildReports ( initBuildReportsFeature ) where -import Distribution.Server.Framework hiding (BuildLog, TestLog, BuildCovg) +import Distribution.Server.Framework hiding (BuildLog, TestLog, TestReportLog, BuildCovg) import Distribution.Server.Features.Users import Distribution.Server.Features.Upload @@ -16,7 +16,7 @@ import Distribution.Server.Features.BuildReports.Backup import qualified Distribution.Server.Features.BuildReports.State as Acid import qualified Distribution.Server.Features.BuildReports.BuildReport as BuildReport import Distribution.Server.Features.BuildReports.BuildReport (BuildReport(..)) -import Distribution.Server.Features.BuildReports.BuildReports (BuildReports, BuildReportId(..), BuildCovg(..), BuildLog(..), TestLog(..)) +import Distribution.Server.Features.BuildReports.BuildReports (BuildReports, BuildReportId(..), BuildCovg(..), BuildLog(..), TestLog(..), TestReportLog(..)) import qualified Distribution.Server.Framework.ResponseContentTypes as Resource import Distribution.Server.Packages.Types @@ -42,13 +42,14 @@ data ReportsFeature = ReportsFeature { reportsFeatureInterface :: HackageFeature, packageReports :: DynamicPath -> ([(BuildReportId, BuildReport)] -> ServerPartE Response) -> ServerPartE Response, - packageReport :: DynamicPath -> ServerPartE (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg), + packageReport :: DynamicPath -> ServerPartE (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog), queryPackageReports :: forall m. MonadIO m => PackageId -> m [(BuildReportId, BuildReport)], queryBuildLog :: forall m. MonadIO m => BuildLog -> m Resource.BuildLog, queryTestLog :: forall m. MonadIO m => TestLog -> m Resource.TestLog, + queryTestReportLog :: forall m. MonadIO m => TestReportLog -> m Resource.TestReportLog, pkgReportDetails :: forall m. MonadIO m => (PackageIdentifier, Bool) -> m BuildReport.PkgDetails, - queryLastReportStats:: forall m. MonadIO m => PackageIdentifier -> m (Maybe (BuildReportId, BuildReport, Maybe BuildCovg)), + queryLastReportStats:: forall m. MonadIO m => PackageIdentifier -> m (Maybe (BuildReportId, BuildReport, Maybe BuildCovg, Maybe TestReportLog)), queryRunTests :: forall m. MonadIO m => PackageId -> m Bool, reportsResource :: ReportsResource } @@ -62,6 +63,7 @@ data ReportsResource = ReportsResource { reportsPage :: Resource, reportsLog :: Resource, reportsTest :: Resource, + reportsTestReport :: Resource, reportsReset:: Resource, reportsTestsEnabled :: Resource, reportsListUri :: String -> PackageId -> String, @@ -124,6 +126,7 @@ buildReportsFeature name , reportsPage , reportsLog , reportsTest + , reportsTestReport , reportsReset , reportsTestsEnabled ] @@ -171,14 +174,19 @@ buildReportsFeature name , resourcePut = [ ("", putBuildLog) ] } , reportsTest = (extendResourcePath "/reports/:id/test" corePackagePage) { - resourceDesc = [ (GET, "Get the test log associated with a build report") - , (DELETE, "Delete a test log") - , (PUT, "Upload a test log for a build report") + resourceDesc = [ (GET, "Get the test build log associated with a build report") + , (DELETE, "Delete a test build log") + , (PUT, "Upload a test build log for a build report") ] , resourceGet = [ ("txt", serveTestLog) ] , resourceDelete = [ ("", deleteTestLog )] , resourcePut = [ ("", putTestLog) ] } + , reportsTestReport = (extendResourcePath "/reports/:id/testReport" corePackagePage) { + resourceDesc = [ (GET, "Get the test report log associated with a build report") + ] + , resourceGet = [ ("txt", serveTestReportLog) ] + } , reportsListUri = \format pkgid -> renderResource (reportsList reportsResource) [display pkgid, format] , reportsPageUri = \format pkgid repid -> renderResource (reportsPage reportsResource) [display pkgid, display repid, format] , reportsLogUri = \pkgid repid -> renderResource (reportsLog reportsResource) [display pkgid, display repid] @@ -199,7 +207,7 @@ buildReportsFeature name guardValidPackageId pkgid queryPackageReports pkgid >>= continue - packageReport :: DynamicPath -> ServerPartE (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg) + packageReport :: DynamicPath -> ServerPartE (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog) packageReport dpath = do pkgid <- packageInPath dpath guardValidPackageId pkgid @@ -207,12 +215,12 @@ buildReportsFeature name mreport <- queryState reportsState $ Acid.LookupReportCovg pkgid reportId case mreport of Nothing -> errNotFound "Report not found" [MText "Build report does not exist"] - Just (report, mlog, mtest, covg) -> return (reportId, report, mlog, mtest, covg) + Just (report, mlog, mtest, covg, testReportLog) -> return (reportId, report, mlog, mtest, covg, testReportLog) queryPackageReports :: MonadIO m => PackageId -> m [(BuildReportId, BuildReport)] queryPackageReports pkgid = do reports <- queryState reportsState $ Acid.LookupPackageReports pkgid - return $ map (second (\(a, _, _) -> a)) reports + return $ map (second (\(a, _, _, _) -> a)) reports queryBuildLog :: MonadIO m => BuildLog -> m Resource.BuildLog queryBuildLog (BuildLog blobId) = do @@ -224,6 +232,11 @@ buildReportsFeature name file <- liftIO $ BlobStorage.fetch store blobId return $ Resource.TestLog file + queryTestReportLog :: MonadIO m => TestReportLog -> m Resource.TestReportLog + queryTestReportLog (TestReportLog blobId) = do + file <- liftIO $ BlobStorage.fetch store blobId + return $ Resource.TestReportLog file + pkgReportDetails :: MonadIO m => (PackageIdentifier, Bool) -> m BuildReport.PkgDetails--(PackageIdentifier, Bool, Maybe (BuildStatus, Maybe UTCTime, Maybe Version)) pkgReportDetails (pkgid, docs) = do failCnt <- queryState reportsState $ Acid.LookupFailCount pkgid @@ -231,17 +244,17 @@ buildReportsFeature name runTests <- fmap Just . queryState reportsState $ Acid.LookupRunTests pkgid (time, ghcId) <- case latestRpt of Nothing -> return (Nothing,Nothing) - Just (_, brp, _, _, _) -> do + Just (_, brp, _, _, _, _) -> do let (CompilerId _ vrsn) = compiler brp return (time brp, Just vrsn) return (BuildReport.PkgDetails pkgid docs failCnt time ghcId runTests) - queryLastReportStats :: MonadIO m => PackageIdentifier -> m (Maybe (BuildReportId, BuildReport, Maybe BuildCovg)) + queryLastReportStats :: MonadIO m => PackageIdentifier -> m (Maybe (BuildReportId, BuildReport, Maybe BuildCovg, Maybe TestReportLog)) queryLastReportStats pkgid = do lookupRes <- queryState reportsState $ Acid.LookupLatestReport pkgid case lookupRes of Nothing -> return Nothing - Just (rptId, rpt, _, _, covg) -> return (Just (rptId, rpt, covg)) + Just (rptId, rpt, _, _, covg, testReportLog) -> return (Just (rptId, rpt, covg, testReportLog)) queryRunTests :: MonadIO m => PackageId -> m Bool queryRunTests pkgid = queryState reportsState $ Acid.LookupRunTests pkgid @@ -251,13 +264,13 @@ buildReportsFeature name textPackageReports dpath = packageReports dpath $ return . toResponse . show textPackageReport dpath = do - (_, report, _, _, _) <- packageReport dpath + (_, report, _, _, _, _) <- packageReport dpath return . toResponse $ BuildReport.show report -- result: not-found error or text file serveBuildLog :: DynamicPath -> ServerPartE Response serveBuildLog dpath = do - (repid, _, mlog, _, _) <- packageReport dpath + (repid, _, mlog, _, _, _) <- packageReport dpath case mlog of Nothing -> errNotFound "Log not found" [MText $ "Build log for report " ++ display repid ++ " not found"] Just logId -> do @@ -267,13 +280,22 @@ buildReportsFeature name -- result: not-found error or text file serveTestLog :: DynamicPath -> ServerPartE Response serveTestLog dpath = do - (repid, _, _, mtest, _) <- packageReport dpath + (repid, _, _, mtest, _, _) <- packageReport dpath case mtest of - Nothing -> errNotFound "Test log not found" [MText $ "Test log for report " ++ display repid ++ " not found"] + Nothing -> errNotFound "Test build log not found" [MText $ "Test build log for report " ++ display repid ++ " not found"] Just logId -> do cacheControlWithoutETag [Public, maxAgeDays 30] toResponse <$> queryTestLog logId + serveTestReportLog :: DynamicPath -> ServerPartE Response + serveTestReportLog dpath = do + (repid, _, _, _, _, mtestReport) <- packageReport dpath + case mtestReport of + Nothing -> errNotFound "Test report log not found" [MText $ "Test report log for report " ++ display repid ++ " not found"] + Just logId -> do + cacheControlWithoutETag [Public, maxAgeDays 30] + toResponse <$> queryTestReportLog logId + -- result: auth error, not-found error, parse error, or redirect submitBuildReport :: DynamicPath -> ServerPartE Response @@ -410,6 +432,7 @@ buildReportsFeature name logBody = BuildReport.logContent buildFiles testBody = BuildReport.testContent buildFiles covgBody = BuildReport.coverageContent buildFiles + testReportBody = BuildReport.testReportContent buildFiles failStatus = BuildReport.buildFail buildFiles updateState reportsState $ Acid.SetFailStatus pkgid failStatus @@ -424,8 +447,9 @@ buildReportsFeature name report' <- liftIO $ BuildReport.affixTimestamp report logBlob <- liftIO $ traverse (\x -> BlobStorage.add store $ fromString x) logBody testBlob <- liftIO $ traverse (\x -> BlobStorage.add store $ fromString x) testBody + testReportBlob <- liftIO $ traverse (\x -> BlobStorage.add store $ fromString x) testReportBody reportId <- updateState reportsState $ - Acid.AddRptLogTestCovg pkgid (report', (fmap BuildLog logBlob), (fmap TestLog testBlob), (fmap BuildReport.parseCovg covgBody)) + Acid.AddRptAllLogsCovg pkgid (report', (fmap BuildLog logBlob), (fmap TestLog testBlob), (fmap BuildReport.parseCovg covgBody), (fmap TestReportLog testReportBlob)) -- redirect to new reports page seeOther (reportsPageUri reportsResource "" pkgid reportId) $ toResponse () diff --git a/src/Distribution/Server/Features/BuildReports/Backup.hs b/src/Distribution/Server/Features/BuildReports/Backup.hs index 09e9fe4c5..18c6c1b55 100644 --- a/src/Distribution/Server/Features/BuildReports/Backup.hs +++ b/src/Distribution/Server/Features/BuildReports/Backup.hs @@ -8,7 +8,7 @@ module Distribution.Server.Features.BuildReports.Backup ( import Distribution.Server.Features.BuildReports.BuildReport (BuildReport) import qualified Distribution.Server.Features.BuildReports.BuildReport as Report -import Distribution.Server.Features.BuildReports.BuildReports (BuildReports(..), BuildCovg(..), PkgBuildReports(..), BuildReportId(..), BuildLog(..), TestLog(..)) +import Distribution.Server.Features.BuildReports.BuildReports (BuildReports(..), BuildCovg(..), PkgBuildReports(..), BuildReportId(..), BuildLog(..), TestLog(..), TestReportLog) import qualified Distribution.Server.Features.BuildReports.BuildReports as Reports import qualified Distribution.Server.Framework.BlobStorage as BlobStorage @@ -94,8 +94,8 @@ packageReportsToExport :: PackageId -> PkgBuildReports -> [BackupEntry] packageReportsToExport pkgId pkgReports = concatMap (uncurry $ reportToExport prefix) (Map.toList $ Reports.reports pkgReports) where prefix = ["package", display pkgId] -reportToExport :: [FilePath] -> BuildReportId -> (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg ) -> [BackupEntry] -reportToExport prefix reportId (report, mlog, _, _) = BackupByteString (getPath ".txt") (packUTF8 $ Report.show report) : +reportToExport :: [FilePath] -> BuildReportId -> (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog) -> [BackupEntry] +reportToExport prefix reportId (report, mlog, _, _, _) = BackupByteString (getPath ".txt") (packUTF8 $ Report.show report) : case mlog of Nothing -> []; Just (BuildLog blobId) -> [blobToBackup (getPath ".log") blobId] where getPath ext = prefix ++ [display reportId ++ ext] diff --git a/src/Distribution/Server/Features/BuildReports/BuildReport.hs b/src/Distribution/Server/Features/BuildReports/BuildReport.hs index c4ddda0a3..fa5f7af8b 100644 --- a/src/Distribution/Server/Features/BuildReports/BuildReport.hs +++ b/src/Distribution/Server/Features/BuildReports/BuildReport.hs @@ -274,6 +274,16 @@ data BuildCovg = BuildCovg { topLevel :: (Int,Int) } deriving (Eq, Show) +instance Arbitrary BuildCovg where + arbitrary = + BuildCovg + <$> intPair + <*> liftA3 BooleanCovg intPair intPair intPair + <*> intPair + <*> intPair + <*> intPair + where intPair = liftA2 (,) arbitrary arbitrary + instance MemSize BuildCovg where memSize (BuildCovg a (BooleanCovg b c d) e f g) = memSize7 a b c d e f g @@ -498,6 +508,10 @@ instance Arbitrary Outcome where data BuildStatus = BuildOK | BuildFailCnt Int deriving (Eq, Ord, Show) + +instance Arbitrary BuildStatus where + arbitrary = oneof [ pure BuildOK, BuildFailCnt <$> arbitrary ] + instance ToJSON BuildStatus where toJSON (BuildFailCnt a) = toJSON a toJSON BuildOK = toJSON ((-1)::Int) @@ -618,6 +632,7 @@ data BuildFiles = BuildFiles { logContent :: Maybe String, testContent :: Maybe String, coverageContent :: Maybe String, + testReportContent :: Maybe String, buildFail :: Bool } deriving Show @@ -628,6 +643,7 @@ instance Data.Aeson.FromJSON BuildFiles where <*> o .:? "log" <*> o .:? "test" <*> o .:? "coverage" + <*> o .:? "testReport" <*> o .: "buildFail" instance Data.Aeson.ToJSON BuildFiles where @@ -636,6 +652,7 @@ instance Data.Aeson.ToJSON BuildFiles where "log" .= logContent p, "test" .= testContent p, "coverage" .= coverageContent p, + "testReport".= testReportContent p, "buildFail" .= buildFail p ] data PkgDetails = PkgDetails { diff --git a/src/Distribution/Server/Features/BuildReports/BuildReports.hs b/src/Distribution/Server/Features/BuildReports/BuildReports.hs index 831c8a925..c926c5e60 100644 --- a/src/Distribution/Server/Features/BuildReports/BuildReports.hs +++ b/src/Distribution/Server/Features/BuildReports/BuildReports.hs @@ -1,14 +1,14 @@ {-# LANGUAGE DeriveDataTypeable, GeneralizedNewtypeDeriving, TemplateHaskell, TypeFamilies #-} -{-# OPTIONS_GHC -fno-warn-orphans #-} module Distribution.Server.Features.BuildReports.BuildReports ( BuildReport(..), BuildReports(..), - BuildReports_v3, BuildReportId(..), + PkgBuildReports_v4(..), PkgBuildReports(..), BuildLog(..), TestLog(..), + TestReportLog(..), BuildCovg(..), BuildStatus(..), addRptLogTestCovg, @@ -53,12 +53,13 @@ import Data.SafeCopy import qualified Data.List as L import qualified Data.Char as Char import Data.Maybe (fromMaybe) +import Test.QuickCheck (Arbitrary(..)) import Text.StringTemplate (ToSElem(..)) newtype BuildReportId = BuildReportId Int - deriving (Eq, Ord, Show, MemSize, Pretty) + deriving (Eq, Ord, Show, MemSize, Pretty, Arbitrary) incrementReportId :: BuildReportId -> BuildReportId incrementReportId (BuildReportId n) = BuildReportId (n+1) @@ -80,9 +81,12 @@ instance Parsec BuildReportId where f c = Char.ord c - Char.ord '0' newtype BuildLog = BuildLog BlobStorage.BlobId - deriving (Eq, Show, MemSize) + deriving (Eq, Show, MemSize, Arbitrary) newtype TestLog = TestLog BlobStorage.BlobId + deriving (Eq, Show, MemSize, Arbitrary) + +newtype TestReportLog = TestReportLog BlobStorage.BlobId deriving (Eq, Show, MemSize) data PkgBuildReports = PkgBuildReports { @@ -90,7 +94,7 @@ data PkgBuildReports = PkgBuildReports { -- perhaps deserving its own data structure (SubmittedReport?) -- When a report was submitted is very useful information. -- also, use IntMap instead of Map BuildReportId? - reports :: !(Map BuildReportId (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg )), + reports :: !(Map BuildReportId (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog)), -- one more than the maximum report id used nextReportId :: !BuildReportId, buildStatus :: !BuildStatus, @@ -115,18 +119,18 @@ emptyReports = BuildReports { reportsIndex = Map.empty } -lookupReport :: PackageId -> BuildReportId -> BuildReports -> Maybe (BuildReport, Maybe BuildLog, Maybe TestLog) +lookupReport :: PackageId -> BuildReportId -> BuildReports -> Maybe (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe TestReportLog) lookupReport pkgid reportId buildReports = remCvg.Map.lookup reportId . reports =<< Map.lookup pkgid (reportsIndex buildReports) where remCvg Nothing = Nothing - remCvg (Just (brpt,blog,btest,_)) = Just (brpt,blog,btest) + remCvg (Just (brpt,blog,btest,_,testReportLog)) = Just (brpt,blog,btest,testReportLog) -lookupPackageReports :: PackageId -> BuildReports -> [(BuildReportId, (BuildReport, Maybe BuildLog, Maybe TestLog))] +lookupPackageReports :: PackageId -> BuildReports -> [(BuildReportId, (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe TestReportLog))] lookupPackageReports pkgid buildReports = case Map.lookup pkgid (reportsIndex buildReports) of Nothing -> [] Just rs -> map removeCovg $ Map.toList (reports rs) where - removeCovg (brid,(brpt,blog,btest,_)) = (brid,(brpt,blog,btest)) + removeCovg (brid,(brpt,blog,btest,_,testReportLog)) = (brid,(brpt,blog,btest,testReportLog)) ------------------------- -- PackageIds should /not/ have empty Versions. Caller should ensure this. @@ -134,7 +138,7 @@ addReport :: PackageId -> (BuildReport, Maybe BuildLog, Maybe TestLog) -> BuildR addReport pkgid (brpt,blog,btest) buildReports = let pkgReports = Map.findWithDefault emptyPkgReports pkgid (reportsIndex buildReports) reportId = nextReportId pkgReports - pkgReports' = PkgBuildReports { reports = Map.insert reportId (brpt,blog,btest,Nothing) (reports pkgReports) + pkgReports' = PkgBuildReports { reports = Map.insert reportId (brpt,blog,btest,Nothing,Nothing) (reports pkgReports) , nextReportId = incrementReportId reportId , buildStatus = buildStatus pkgReports , runTests = runTests pkgReports } @@ -143,7 +147,7 @@ addReport pkgid (brpt,blog,btest) buildReports = unsafeSetReport :: PackageId -> BuildReportId -> (BuildReport, Maybe BuildLog) -> BuildReports -> BuildReports unsafeSetReport pkgid reportId (brpt,blog) buildReports = let pkgReports = Map.findWithDefault emptyPkgReports pkgid (reportsIndex buildReports) - pkgReports' = PkgBuildReports { reports = Map.insert reportId (brpt,blog,Nothing,Nothing) (reports pkgReports) + pkgReports' = PkgBuildReports { reports = Map.insert reportId (brpt,blog,Nothing,Nothing,Nothing) (reports pkgReports) , nextReportId = max (incrementReportId reportId) (nextReportId pkgReports) , buildStatus = buildStatus pkgReports , runTests = runTests pkgReports } @@ -162,7 +166,7 @@ setBuildLog pkgid reportId buildLog buildReports = case Map.lookup pkgid (report Nothing -> Nothing Just pkgReports -> case Map.lookup reportId (reports pkgReports) of Nothing -> Nothing - Just (rlog, _, btest, covg) -> let pkgReports' = pkgReports { reports = Map.insert reportId (rlog, buildLog, btest, covg) (reports pkgReports) } + Just (rlog, _, btest, covg, testReportLog) -> let pkgReports' = pkgReports { reports = Map.insert reportId (rlog, buildLog, btest, covg, testReportLog) (reports pkgReports) } in Just $ buildReports { reportsIndex = Map.insert pkgid pkgReports' (reportsIndex buildReports) } setTestLog :: PackageId -> BuildReportId -> Maybe TestLog -> BuildReports -> Maybe BuildReports @@ -170,10 +174,10 @@ setTestLog pkgid reportId testLog buildReports = case Map.lookup pkgid (reportsI Nothing -> Nothing Just pkgReports -> case Map.lookup reportId (reports pkgReports) of Nothing -> Nothing - Just (rlog, blog, _, covg) -> let pkgReports' = pkgReports { reports = Map.insert reportId (rlog, blog, testLog, covg) (reports pkgReports) } + Just (rlog, blog, _, covg, testReportLog) -> let pkgReports' = pkgReports { reports = Map.insert reportId (rlog, blog, testLog, covg, testReportLog) (reports pkgReports) } in Just $ buildReports { reportsIndex = Map.insert pkgid pkgReports' (reportsIndex buildReports) } -addRptLogTestCovg :: PackageId -> (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg ) -> BuildReports -> (BuildReports, BuildReportId) +addRptLogTestCovg :: PackageId -> (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog ) -> BuildReports -> (BuildReports, BuildReportId) addRptLogTestCovg pkgid report buildReports = let pkgReports = Map.findWithDefault emptyPkgReports pkgid (reportsIndex buildReports) reportId = nextReportId pkgReports @@ -183,7 +187,7 @@ addRptLogTestCovg pkgid report buildReports = , runTests = runTests pkgReports } in (buildReports { reportsIndex = Map.insert pkgid pkgReports' (reportsIndex buildReports) }, reportId) -lookupReportCovg :: PackageId -> BuildReportId -> BuildReports -> Maybe (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg ) +lookupReportCovg :: PackageId -> BuildReportId -> BuildReports -> Maybe (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog) lookupReportCovg pkgid reportId buildReports = Map.lookup reportId . reports =<< Map.lookup pkgid (reportsIndex buildReports) setFailStatus :: PackageId -> Bool -> BuildReports -> BuildReports @@ -216,15 +220,15 @@ lookupFailCount pkgid buildReports = do rp <- Map.lookup pkgid (reportsIndex buildReports) return $ buildStatus rp -lookupLatestReport :: PackageId -> BuildReports -> Maybe (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg) +lookupLatestReport :: PackageId -> BuildReports -> Maybe (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog) lookupLatestReport pkgid buildReports = do rp <- Map.lookup pkgid (reportsIndex buildReports) let rs = reports rp - (maxKey, (rep, buildLog, testLog, covg)) <- + (maxKey, (rep, buildLog, testLog, covg, testReportLog)) <- if Map.null rs then Nothing else Just $ Map.findMax rs - Just (maxKey, rep, buildLog, testLog, covg) + Just (maxKey, rep, buildLog, testLog, covg, testReportLog) lookupRunTests :: PackageId -> BuildReports -> Bool lookupRunTests pkgid buildReports = maybe True runTests $ Map.lookup pkgid (reportsIndex buildReports) @@ -274,13 +278,14 @@ instance Migrate BuildLog where deriveSafeCopy 2 'extension ''BuildLog deriveSafeCopy 0 'base ''TestLog +deriveSafeCopy 0 'base ''TestReportLog -- note: if the set of report ids is [1, 2, 3], then nextReportId = 4 -- after calling deleteReport for 3, the set is [1, 2] and nextReportId is still 4. -- however, upon importing, nextReportId will = 3, one more than the maximum present -- this is also a problem in ReportsBackup.hs. but it's not a major issue I think. instance SafeCopy PkgBuildReports where - version = 4 + version = 5 kind = extension putCopy (PkgBuildReports x _ y z) = contain $ safePut (x,y,z) getCopy = contain $ mkReports <$> safeGet @@ -294,6 +299,31 @@ instance SafeCopy PkgBuildReports where instance MemSize PkgBuildReports where memSize (PkgBuildReports a b c d) = memSize4 a b c d +data PkgBuildReports_v4 = PkgBuildReports_v4 { + reports_v4 :: !(Map BuildReportId (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg)), + nextReportId_v4 :: !BuildReportId, + buildStatus_v4 :: !BuildStatus, + runTests_v4 :: !Bool +} deriving (Eq, Show) + +instance Arbitrary PkgBuildReports_v4 where + arbitrary = PkgBuildReports_v4 <$> arbitrary <*> arbitrary + <*> arbitrary <*> arbitrary + +instance SafeCopy PkgBuildReports_v4 where + version = 4 + kind = extension + putCopy (PkgBuildReports_v4 x _ y z) = contain $ safePut (x,y,z) + getCopy = contain $ mkReports <$> safeGet + where + mkReports (rs,f,b) = PkgBuildReports_v4 rs + (if Map.null rs + then BuildReportId 1 + else incrementReportId (fst $ Map.findMax rs)) + f b + +instance MemSize PkgBuildReports_v4 where + memSize (PkgBuildReports_v4 a b c d) = memSize4 a b c d data PkgBuildReports_v3 = PkgBuildReports_v3 { reports_v3 :: !(Map BuildReportId (BuildReport, Maybe BuildLog, Maybe BuildCovg )), @@ -373,16 +403,26 @@ instance Migrate PkgBuildReports_v3 where migrateMap = Map.mapKeys (\x->x) . Map.map (\(br, l) -> (br, l, Nothing)) -instance Migrate PkgBuildReports where - type MigrateFrom PkgBuildReports = PkgBuildReports_v3 +instance Migrate PkgBuildReports_v4 where + type MigrateFrom PkgBuildReports_v4 = PkgBuildReports_v3 migrate (PkgBuildReports_v3 m n o) = - PkgBuildReports (migrateMap m) n o True + PkgBuildReports_v4 (migrateMap m) n o True where migrateMap :: Map BuildReportId (BuildReport, Maybe BuildLog, Maybe BuildCovg) -> Map BuildReportId (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg) migrateMap = Map.mapKeys id . Map.map (\(br, l, c) -> (br, l, Nothing, c)) +instance Migrate PkgBuildReports where + type MigrateFrom PkgBuildReports = PkgBuildReports_v4 + migrate (PkgBuildReports_v4 m n o rntsts_v4) = + PkgBuildReports (migrateMap m) n o rntsts_v4 + where + migrateMap :: Map BuildReportId (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg) + -> Map BuildReportId (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog) + migrateMap = Map.mapKeys id + . Map.map (\(br, l, c, covg) -> (br, l, c, covg, Nothing)) + data BuildReports_v0 = BuildReports_v0 !(Map.Map PackageIdentifier_v0 PkgBuildReports_v0) @@ -422,12 +462,26 @@ instance MemSize BuildReports_v3 where deriveSafeCopy 3 'extension ''BuildReports_v3 -instance Migrate BuildReports where - type MigrateFrom BuildReports = BuildReports_v3 +data BuildReports_v4 = BuildReports_v4 + { reportsIndex_v4 :: !(Map.Map PackageId PkgBuildReports_v4) + } deriving (Eq, Show) + +instance Migrate BuildReports_v4 where + type MigrateFrom BuildReports_v4 = BuildReports_v3 migrate (BuildReports_v3 m) = + BuildReports_v4 (Map.mapKeys id $ Map.map migrate m) + +instance MemSize BuildReports_v4 where + memSize (BuildReports_v4 a) = memSize1 a + +deriveSafeCopy 4 'extension ''BuildReports_v4 + +instance Migrate BuildReports where + type MigrateFrom BuildReports = BuildReports_v4 + migrate (BuildReports_v4 m) = BuildReports (Map.mapKeys id $ Map.map migrate m) instance MemSize BuildReports where memSize (BuildReports a) = memSize1 a -deriveSafeCopy 4 'extension ''BuildReports +deriveSafeCopy 5 'extension ''BuildReports diff --git a/src/Distribution/Server/Features/BuildReports/State.hs b/src/Distribution/Server/Features/BuildReports/State.hs index 0dec1518c..5aae2a98e 100644 --- a/src/Distribution/Server/Features/BuildReports/State.hs +++ b/src/Distribution/Server/Features/BuildReports/State.hs @@ -5,7 +5,7 @@ module Distribution.Server.Features.BuildReports.State where import Distribution.Server.Features.BuildReports.BuildReports - (BuildReportId, BuildLog, TestLog, BuildReport, BuildReports,BuildCovg, BuildStatus) + (BuildReportId, BuildLog, TestLog, TestReportLog, BuildReport, BuildReports,BuildCovg, BuildStatus) import qualified Distribution.Server.Features.BuildReports.BuildReports as BuildReports import Distribution.Package @@ -39,10 +39,10 @@ deleteReport pkgid reportId = do Nothing -> return False Just reports -> State.put reports >> return True -lookupReport :: PackageId -> BuildReportId -> Query BuildReports (Maybe (BuildReport, Maybe BuildLog, Maybe TestLog)) +lookupReport :: PackageId -> BuildReportId -> Query BuildReports (Maybe (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe TestReportLog)) lookupReport pkgid reportId = asks (BuildReports.lookupReport pkgid reportId) -lookupPackageReports :: PackageId -> Query BuildReports [(BuildReportId, (BuildReport, Maybe BuildLog, Maybe TestLog))] +lookupPackageReports :: PackageId -> Query BuildReports [(BuildReportId, (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe TestReportLog))] lookupPackageReports pkgid = asks (BuildReports.lookupPackageReports pkgid) getBuildReports :: Query BuildReports BuildReports @@ -51,14 +51,15 @@ getBuildReports = ask replaceBuildReports :: BuildReports -> Update BuildReports () replaceBuildReports = State.put -addRptLogCovg :: PackageId -> (BuildReport, Maybe BuildLog, Maybe BuildCovg ) -> Update BuildReports BuildReportId +-- | Superseded by 'addRptAllLogsCovg'. +addRptLogCovg :: PackageId -> (BuildReport, Maybe BuildLog, Maybe BuildCovg) -> Update BuildReports BuildReportId addRptLogCovg pkgid (bRpt, blog, bcovg) = do buildReports <- State.get - let (reports, reportId) = BuildReports.addRptLogTestCovg pkgid (bRpt, blog, Nothing, bcovg) buildReports + let (reports, reportId) = BuildReports.addRptLogTestCovg pkgid (bRpt, blog, Nothing, bcovg, Nothing) buildReports State.put reports return reportId -lookupReportCovg :: PackageId -> BuildReportId -> Query BuildReports (Maybe (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg)) +lookupReportCovg :: PackageId -> BuildReportId -> Query BuildReports (Maybe (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog)) lookupReportCovg pkgid reportId = asks (BuildReports.lookupReportCovg pkgid reportId) setFailStatus :: PackageId -> Bool -> Update BuildReports () @@ -77,13 +78,21 @@ resetFailCount pkgid = do lookupFailCount :: PackageId -> Query BuildReports (Maybe BuildStatus) lookupFailCount pkgid = asks (BuildReports.lookupFailCount pkgid) -lookupLatestReport :: PackageId -> Query BuildReports (Maybe (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg)) +lookupLatestReport :: PackageId -> Query BuildReports (Maybe (BuildReportId, BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog)) lookupLatestReport pkgid = asks (BuildReports.lookupLatestReport pkgid) -addRptLogTestCovg :: PackageId -> (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg ) -> Update BuildReports BuildReportId +-- | Superseded by 'addRptAllLogsCovg'. +addRptLogTestCovg :: PackageId -> (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg) -> Update BuildReports BuildReportId addRptLogTestCovg pkgid (bRpt, blog, btest, bcovg) = do buildReports <- State.get - let (reports, reportId) = BuildReports.addRptLogTestCovg pkgid (bRpt, blog, btest, bcovg) buildReports + let (reports, reportId) = BuildReports.addRptLogTestCovg pkgid (bRpt, blog, btest, bcovg, Nothing) buildReports + State.put reports + return reportId + +addRptAllLogsCovg :: PackageId -> (BuildReport, Maybe BuildLog, Maybe TestLog, Maybe BuildCovg, Maybe TestReportLog) -> Update BuildReports BuildReportId +addRptAllLogsCovg pkgid (bRpt, blog, btest, bcovg, testReportLog) = do + buildReports <- State.get + let (reports, reportId) = BuildReports.addRptLogTestCovg pkgid (bRpt, blog, btest, bcovg, testReportLog) buildReports State.put reports return reportId @@ -118,6 +127,7 @@ makeAcidic ''BuildReports ['addReport ,'lookupFailCount ,'lookupLatestReport ,'addRptLogTestCovg + ,'addRptAllLogsCovg ,'setTestLog ,'lookupRunTests ,'setRunTests diff --git a/src/Distribution/Server/Features/Html.hs b/src/Distribution/Server/Features/Html.hs index a5b031980..9fffd0f39 100644 --- a/src/Distribution/Server/Features/Html.hs +++ b/src/Distribution/Server/Features/Html.hs @@ -669,10 +669,10 @@ mkHtmlCore ServerEnv{serverBaseURI, serverBlobStore} userDb <- queryGetUserDb maintainerlist <- liftIO $ queryUserGroup maintainers let - idAndReport = fmap (\(rptId, rpt, _) -> (rptId, rpt)) rptStats + idAndReport = fmap (\(rptId, rpt, _, _) -> (rptId, rpt)) rptStats install = getInstall $ fmap (fst &&& BR.installOutcome . snd) idAndReport test = getTest $ fmap ( BR.testsOutcome . snd) idAndReport - covg = getAvgCovg $ (\(_, _, cvg) -> cvg) =<< rptStats + covg = getAvgCovg $ (\(_, _, cvg, _) -> cvg) =<< rptStats loadDocMeta | Just doctarblob <- mdoctarblob , Just docIndex <- mdocIndex @@ -1114,9 +1114,10 @@ mkHtmlReports HtmlUtilities{..} CoreFeature{..} UploadFeature{..} UserFeature{.. servePackageReport :: DynamicPath -> ServerPartE Response servePackageReport dpath = do - (repid, report, mlog, mtest, covg) <- packageReport dpath + (repid, report, mlog, mtest, covg, testReportLog) <- packageReport dpath mlog' <- traverse queryBuildLog mlog mtest' <- traverse queryTestLog mtest + testReportLog' <- traverse queryTestReportLog testReportLog let covg' = fmap getCvgDet covg pkgid <- packageInPath dpath cacheControlWithoutETag [Public, maxAgeDays 30] @@ -1126,6 +1127,7 @@ mkHtmlReports HtmlUtilities{..} CoreFeature{..} UploadFeature{..} UserFeature{.. , "report" $= (repid, report) , "log" $= toMessage <$> mlog' , "test" $= toMessage <$> mtest' + , "testReport" $= toMessage <$> testReportLog' , "covg" $= covg' ] where diff --git a/src/Distribution/Server/Features/Security/MD5.hs b/src/Distribution/Server/Features/Security/MD5.hs index 02c4aa05c..72331cf86 100644 --- a/src/Distribution/Server/Features/Security/MD5.hs +++ b/src/Distribution/Server/Features/Security/MD5.hs @@ -27,6 +27,7 @@ import qualified Data.ByteString.Char8 as BS.Char8 import qualified Data.ByteString.Lazy as BS.Lazy import Data.SafeCopy import qualified Data.Serialize as Ser +import Test.QuickCheck (Arbitrary(..)) -- cryptohash import qualified Crypto.Hash.MD5 as MD5 @@ -39,6 +40,9 @@ import Distribution.Server.Util.ReadDigest data MD5Digest = MD5Digest {-# UNPACK #-} !Word64 {-# UNPACK #-} !Word64 deriving (Eq,Ord) +instance Arbitrary MD5Digest where + arbitrary = MD5Digest <$> arbitrary <*> arbitrary + instance NFData MD5Digest where rnf !_ = () -- 'MD5Digest' has only strict primitive fields, hence WHNF==NF diff --git a/src/Distribution/Server/Framework/BlobStorage.hs b/src/Distribution/Server/Framework/BlobStorage.hs index 4124d0d07..5e29109a6 100644 --- a/src/Distribution/Server/Framework/BlobStorage.hs +++ b/src/Distribution/Server/Framework/BlobStorage.hs @@ -53,6 +53,7 @@ import System.Directory import System.IO import Data.Aeson import System.Posix.Files as Posix (createLink) +import Test.QuickCheck (Arbitrary) -- For fsync import System.Posix.Types (Fd(..)) @@ -71,7 +72,7 @@ import System.Posix.IO ( -- | An id for a blob. The content of the blob is stable. -- newtype BlobId = BlobId MD5Digest - deriving (Eq, Ord, Show, MemSize) + deriving (Eq, Ord, Show, MemSize, Arbitrary) instance ToJSON BlobId where toJSON = toJSON . blobMd5 diff --git a/src/Distribution/Server/Framework/MemSize.hs b/src/Distribution/Server/Framework/MemSize.hs index d98e00085..0a92c1366 100644 --- a/src/Distribution/Server/Framework/MemSize.hs +++ b/src/Distribution/Server/Framework/MemSize.hs @@ -159,6 +159,9 @@ instance (MemSize a, MemSize b, MemSize c) => MemSize (a,b,c) where instance (MemSize a, MemSize b, MemSize c, MemSize d) => MemSize (a,b,c,d) where memSize (a,b,c,d) = memSize4 a b c d +instance (MemSize a, MemSize b, MemSize c, MemSize d, MemSize e) => MemSize (a,b,c,d,e) where + memSize (a,b,c,d,e) = memSize5 a b c d e + instance MemSize a => MemSize (Maybe a) where memSize Nothing = memSize0 memSize (Just a) = memSize1 a diff --git a/src/Distribution/Server/Framework/ResponseContentTypes.hs b/src/Distribution/Server/Framework/ResponseContentTypes.hs index fd20d7fe2..32022478b 100644 --- a/src/Distribution/Server/Framework/ResponseContentTypes.hs +++ b/src/Distribution/Server/Framework/ResponseContentTypes.hs @@ -186,6 +186,12 @@ instance ToMessage TestLog where toContentType _ = "text/plain" toMessage (TestLog bs) = bs +newtype TestReportLog = TestReportLog BS.Lazy.ByteString + +instance ToMessage TestReportLog where + toContentType _ = "text/plain" + toMessage (TestReportLog bs) = bs + newtype BuildCovg = BuildCovg BS.Lazy.ByteString instance ToMessage BuildCovg where