<?xml version="1.0"?><ownershipDocument><schemaVersion>X0201</schemaVersion>    <documentType>4</documentType><periodOfReport>2003-10-01</periodOfReport><notSubjectToSection16>0</notSubjectToSection16><issuer><issuerCik>0001028985</issuerCik><issuerName>RAYOVAC CORP</issuerName>        <issuerTradingSymbol>ROV</issuerTradingSymbol></issuer><reportingOwner><reportingOwnerId><rptOwnerCik>0001173421</rptOwnerCik><rptOwnerName>THOMAS BARBARA S</rptOwnerName></reportingOwnerId><reportingOwnerAddress><rptOwnerStreet1>C/O RAYOVAC CORPORATION</rptOwnerStreet1><rptOwnerStreet2>601 RAYOVAC DRIVE</rptOwnerStreet2><rptOwnerCity>MADISON</rptOwnerCity><rptOwnerState>WI</rptOwnerState><rptOwnerZipCode>53711</rptOwnerZipCode><rptOwnerStateDescription></rptOwnerStateDescription></reportingOwnerAddress><reportingOwnerRelationship><isDirector>1</isDirector><isOfficer>0</isOfficer><isTenPercentOwner>0</isTenPercentOwner><isOther>0</isOther></reportingOwnerRelationship></reportingOwner><derivativeTable><derivativeTransaction><securityTitle><value>Stock Option (Right to Buy)</value></securityTitle><conversionOrExercisePrice><value>14.6</value></conversionOrExercisePrice><transactionDate><value>2003-10-01</value></transactionDate><deemedExecutionDate></deemedExecutionDate><transactionCoding><transactionFormType>4</transactionFormType><transactionCode>A</transactionCode><equitySwapInvolved>0</equitySwapInvolved></transactionCoding><transactionTimeliness><value></value></transactionTimeliness><transactionAmounts><transactionShares><value>5000</value><footnoteId id="F1" /></transactionShares><transactionPricePerShare><value>0</value></transactionPricePerShare><transactionAcquiredDisposedCode><value>A</value></transactionAcquiredDisposedCode></transactionAmounts><exerciseDate><footnoteId id="F2" /></exerciseDate><expirationDate><value>2013-10-01</value></expirationDate><underlyingSecurity><underlyingSecurityTitle><value>Common Stock</value></underlyingSecurityTitle><underlyingSecurityShares><value>5000</value></underlyingSecurityShares></underlyingSecurity><postTransactionAmounts><sharesOwnedFollowingTransaction><value>5000</value></sharesOwnedFollowingTransaction></postTransactionAmounts><ownershipNature><directOrIndirectOwnership><value>D</value></directOrIndirectOwnership></ownershipNature></derivativeTransaction></derivativeTable><footnotes><footnote id="F1">Granted option to purchase 5,000 shares of Common Stock.</footnote><footnote id="F2">Exercisable Immediately.</footnote></footnotes><remarks></remarks><ownerSignature><signatureName>/s/Tracy S. Wrycha, as attorney-in-fact</signatureName><signatureDate>2003-10-03</signatureDate></ownerSignature></ownershipDocument>

