<?xml version="1.0"?><ownershipDocument><schemaVersion>X0201</schemaVersion>    <documentType>4</documentType><periodOfReport>2003-09-10</periodOfReport><notSubjectToSection16>0</notSubjectToSection16><issuer><issuerCik>0000314727</issuerCik><issuerName>HANDLEMAN CO /MI/</issuerName>        <issuerTradingSymbol>HDL</issuerTradingSymbol></issuer><reportingOwner><reportingOwnerId><rptOwnerCik>0001248029</rptOwnerCik><rptOwnerName>REUSS LLOYD E</rptOwnerName></reportingOwnerId><reportingOwnerAddress><rptOwnerStreet1>500 KIRTS BOULEVARD</rptOwnerStreet1><rptOwnerStreet2></rptOwnerStreet2><rptOwnerCity>TROY</rptOwnerCity><rptOwnerState>MI</rptOwnerState><rptOwnerZipCode>48084</rptOwnerZipCode><rptOwnerStateDescription></rptOwnerStateDescription></reportingOwnerAddress><reportingOwnerRelationship><isDirector>1</isDirector><isOfficer>0</isOfficer><isTenPercentOwner>0</isTenPercentOwner><isOther>0</isOther></reportingOwnerRelationship></reportingOwner><nonDerivativeTable><nonDerivativeTransaction><securityTitle><value>Common stock one cent par value</value></securityTitle><transactionDate><value>2003-09-10</value></transactionDate><deemedExecutionDate></deemedExecutionDate><transactionCoding><transactionFormType>4</transactionFormType><transactionCode>A</transactionCode><equitySwapInvolved>0</equitySwapInvolved></transactionCoding><transactionTimeliness><value></value></transactionTimeliness><transactionAmounts><transactionShares><value>804</value><footnoteId id="F1" /></transactionShares><transactionPricePerShare><value>15.54</value></transactionPricePerShare><transactionAcquiredDisposedCode><value>A</value></transactionAcquiredDisposedCode></transactionAmounts><postTransactionAmounts><sharesOwnedFollowingTransaction><value>11043</value></sharesOwnedFollowingTransaction></postTransactionAmounts><ownershipNature><directOrIndirectOwnership><value>D</value></directOrIndirectOwnership></ownershipNature></nonDerivativeTransaction></nonDerivativeTable><derivativeTable><derivativeTransaction><securityTitle><value>Director stock option (right to buy)</value></securityTitle><conversionOrExercisePrice><value>15.54</value></conversionOrExercisePrice><transactionDate><value>2003-09-10</value></transactionDate><deemedExecutionDate></deemedExecutionDate><transactionCoding><transactionFormType>4</transactionFormType><transactionCode>A</transactionCode><equitySwapInvolved>0</equitySwapInvolved></transactionCoding><transactionTimeliness><value></value></transactionTimeliness><transactionAmounts><transactionShares><value>2500</value></transactionShares><transactionPricePerShare><value>15.54</value><footnoteId id="F3" /></transactionPricePerShare><transactionAcquiredDisposedCode><value>A</value></transactionAcquiredDisposedCode></transactionAmounts><exerciseDate><footnoteId id="F2" /></exerciseDate><expirationDate><value>2013-09-10</value></expirationDate><underlyingSecurity><underlyingSecurityTitle><value>Common stock one cent par value</value></underlyingSecurityTitle><underlyingSecurityShares><value>2500</value></underlyingSecurityShares></underlyingSecurity><postTransactionAmounts><sharesOwnedFollowingTransaction><value>2500</value></sharesOwnedFollowingTransaction></postTransactionAmounts><ownershipNature><directOrIndirectOwnership><value>D</value></directOrIndirectOwnership></ownershipNature></derivativeTransaction></derivativeTable><footnotes><footnote id="F1">Grant in payment of part of the September 2003 through August 2004 Director compensation. The shares vest 100% on September 1, 2004.</footnote><footnote id="F2">The option vests in three equal annual installments beginning on September 1, 2004.</footnote><footnote id="F3">The option exercise price is $15.54 per share.</footnote></footnotes><remarks></remarks><ownerSignature><signatureName>Lloyd E. Reuss by Kenneth P. Kartje Attorney-In-Fact</signatureName><signatureDate>2003-09-12</signatureDate></ownerSignature></ownershipDocument>

