-----BEGIN PRIVACY-ENHANCED MESSAGE-----
Proc-Type: 2001,MIC-CLEAR
Originator-Name: webmaster@www.sec.gov
Originator-Key-Asymmetric:
 MFgwCgYEVQgBAQICAf8DSgAwRwJAW2sNKK9AVtBzYZmr6aGjlWyK3XmZv3dTINen
 TWSM7vrzLADbmYQaionwg5sDW3P6oaM5D3tdezXMm7z1T+B+twIDAQAB
MIC-Info: RSA-MD5,RSA,
 COq/9Zo3FLfjPB12DNxzvGS1+D6NAAlZcxCSMhJI0uON91+hCMVYbn2dAz6evQ0Q
 DVLRn3z32fNwlr2OKLl9jg==

<SEC-DOCUMENT>0001116679-04-001571.txt : 20040708
<SEC-HEADER>0001116679-04-001571.hdr.sgml : 20040708
<ACCEPTANCE-DATETIME>20040708164804
ACCESSION NUMBER:		0001116679-04-001571
CONFORMED SUBMISSION TYPE:	4
PUBLIC DOCUMENT COUNT:		1
CONFORMED PERIOD OF REPORT:	20040701
FILED AS OF DATE:		20040708

REPORTING-OWNER:	

	OWNER DATA:	
		COMPANY CONFORMED NAME:			HUSIC FRANK J
		CENTRAL INDEX KEY:			0000918220
		STATE OF INCORPORATION:			CA
		FISCAL YEAR END:			1231

	FILING VALUES:
		FORM TYPE:		4
		SEC ACT:		1934 Act
		SEC FILE NUMBER:	000-15898
		FILM NUMBER:		04906501

	BUSINESS ADDRESS:	
		STREET 1:		555 CALIFORNIA STREET, SUITE 2900
		CITY:			SAN FRANCISCO
		STATE:			CA
		ZIP:			94104
		BUSINESS PHONE:		4153980800

ISSUER:		

	COMPANY DATA:	
		COMPANY CONFORMED NAME:			CASUAL MALE RETAIL GROUP INC
		CENTRAL INDEX KEY:			0000813298
		STANDARD INDUSTRIAL CLASSIFICATION:	RETAIL-FAMILY CLOTHING STORES [5651]
		IRS NUMBER:				042623104
		STATE OF INCORPORATION:			DE
		FISCAL YEAR END:			0131

	BUSINESS ADDRESS:	
		STREET 1:		555 TURNPIKE STREET
		CITY:			CANTON
		STATE:			MA
		ZIP:			02021
		BUSINESS PHONE:		7818215900

	MAIL ADDRESS:	
		STREET 1:		555 TURNPIKE STREET
		CITY:			CANTON
		STATE:			MA
		ZIP:			02021

	FORMER COMPANY:	
		FORMER CONFORMED NAME:	DESIGNS INC
		DATE OF NAME CHANGE:	19920703
</SEC-HEADER>
<DOCUMENT>
<TYPE>4
<SEQUENCE>1
<FILENAME>edgar.xml
<DESCRIPTION>PRIMARY DOCUMENT
<TEXT>
<XML>
<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0202</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2004-07-01</periodOfReport>

    <issuer>
        <issuerCik>0000813298</issuerCik>
        <issuerName>CASUAL MALE RETAIL GROUP INC</issuerName>
        <issuerTradingSymbol>CMRG</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0000918220</rptOwnerCik>
            <rptOwnerName>HUSIC FRANK J</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>C/O CASUAL MALE RETAIL GROUP, INC.</rptOwnerStreet1>
            <rptOwnerStreet2>555 TURNPIKE STREET</rptOwnerStreet2>
            <rptOwnerCity>CANTON,</rptOwnerCity>
            <rptOwnerState>MA</rptOwnerState>
            <rptOwnerZipCode>02021</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock, $.01 par value</value>
            </securityTitle>
            <transactionDate>
                <value>2004-07-01</value>
            </transactionDate>
            <transactionCoding>
                <transactionFormType>5</transactionFormType>
                <transactionCode>J</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
                <footnoteId id="F1"/>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>2736</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>0</value>
                    <footnoteId id="F2"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>23748</value>
                    <footnoteId id="F3"/>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Husic Capital Management</value>
                    <footnoteId id="F4"/>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
    </nonDerivativeTable>

    <footnotes>
        <footnote id="F1">This filing reflects a change in the pecuniary interest attributable to the Reporting Person.  Such change arises due to changes in the ownership composition of the private investment funds for which Husic Capital Management, Frank J. Husic &amp; Co., and/or the Reporting Person serve as general partner and/or investment adviser.  The change is not due to transactions by private investment funds or the Reporting Person in the issuer's securities nor due to an affirmative, volitional act by the Reporting Person.</footnote>
        <footnote id="F2">The price of the securities acquired is the proportionate share of each cost lot held by the private investment funds.</footnote>
        <footnote id="F3">Reflect shares for which the Reporting Person has a pecuniary interest.</footnote>
        <footnote id="F4">By private investment funds for which Husic Capital Management, Frank J. Husic &amp; Co., and/or the Reporting Person serve as general partner and/or investment adviser.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>Frank J. Husic</signatureName>
        <signatureDate>2004-07-07</signatureDate>
    </ownerSignature>
</ownershipDocument>

</XML>
</TEXT>
</DOCUMENT>
</SEC-DOCUMENT>
-----END PRIVACY-ENHANCED MESSAGE-----